CNDB-14144: Change Version.LATEST to ED. Move testIndexMetaForNumRows from BM25Test to FeaturesVersionSupportTest and extend coverage for all versions. Add FeaturesVersionSupportEDTest. Make BM25Test tests run on all sai versions post BM25_EARLIEST

Author: ekaterinadimitrova2

src/java/org/apache/cassandra/index/sai/disk/format/Version.java

@@ -74,7 +74,7 @@ public class Version implements Comparable<Version>
     public static final Version VECTOR_EARLIEST = BA;
     public static final Version JVECTOR_EARLIEST = CA;
     public static final Version BM25_EARLIEST = EC;
-    public static final Version LATEST = EC;
+    public static final Version LATEST = ALL.get(0);
     // The current version can be configured to be an earlier version to support partial upgrades that don't
     // write newer versions of the on-disk formats. This is volatile rather than final so that tests may
     // use reflection to change it and safely publish across threads.

test/distributed/org/apache/cassandra/distributed/test/sai/features/FeaturesVersionSupportEDTest.java

@@ -0,0 +1,35 @@
+/*
+ * Copyright DataStax, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.distributed.test.sai.features;
+
+import java.io.IOException;
+
+import org.junit.BeforeClass;
+
+import org.apache.cassandra.index.sai.disk.format.Version;
+
+/**
+ * {@link FeaturesVersionSupportTester} for {@link Version#ED}.
+ */
+public class FeaturesVersionSupportEDTest extends FeaturesVersionSupportTester
+{
+    @BeforeClass
+    public static void setup() throws IOException
+    {
+        initCluster(Version.ED);
+    }
+}

test/unit/org/apache/cassandra/index/sai/cql/BM25Test.java

@@ -16,24 +16,15 @@
 
 package org.apache.cassandra.index.sai.cql;
 
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.List;
+import java.util.*;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Executors;
 import java.util.concurrent.Future;
 import java.util.regex.Pattern;
 import java.util.stream.Collectors;
-import java.util.stream.IntStream;
 
-import org.apache.cassandra.index.sai.SSTableIndex;
-import org.apache.cassandra.index.sai.memory.MemtableIndex;
-import org.apache.cassandra.index.sai.memory.TrieMemtableIndex;
 import org.assertj.core.api.Assertions;
 
-import org.junit.Assert;
 import org.junit.Before;
 import org.junit.Test;
 
@@ -63,11 +54,13 @@ public class BM25Test extends SAITester
     @Parameterized.Parameters(name = "version={0}")
     public static List<Object> data()
     {
-        return Arrays.asList(new Object[]{ Version.BM25_EARLIEST, Version.ED });
+        return Version.ALL.stream().filter(v -> v.onOrAfter(Version.BM25_EARLIEST))
+                                   .map(v -> new Object[]{ v })
+                                   .collect(Collectors.toList());
     }
 
     // Pattern that treats apostrophes within words as part of the word
-    private static final Pattern PATTERN = Pattern.compile("[^\\w']+|'(?=\\s)|(?<=\\s)'");
+    public static final Pattern PATTERN = Pattern.compile("[^\\w']+|'(?=\\s)|(?<=\\s)'");
     public static final int DATASET_BODY_COLUMN = 3;
 
     @Before
@@ -755,7 +748,7 @@ public void testCollections() throws Throwable
         createIndex("CREATE CUSTOM INDEX ON %s (category) USING 'StorageAttachedIndex'");
         createIndex("CREATE CUSTOM INDEX ON %s (map_category) USING 'StorageAttachedIndex'");
         createIndex("CREATE CUSTOM INDEX ON %s (KEYS(map_body)) USING 'StorageAttachedIndex'");
-        insertCollectionData();
+        insertCollectionData(this);
         analyzeDataset("climate");
         analyzeDataset("health");
 
@@ -841,110 +834,7 @@ public void testOrderingSeveralSegments() throws Throwable
                 "climate");
     }
 
-    /**
-     * Asserts that memtable SAI index maintains expected row count, which is, then,
-     * used to store row count in SSTable SAI index and its segments. This is also
-     * asserted.
-     */
-    @Test
-    public void testIndexMetaForNumRows()
-    {
-        SAIUtil.setCurrentVersion(Version.ED);
-
-        createTable("CREATE TABLE %s (id int PRIMARY KEY, category text, score int, " +
-                    "title text, body text, bodyset set<text>, " +
-                    "map_category map<int, text>, map_body map<text, text>)");
-        String bodyIndexName = createAnalyzedIndex("body", true);
-        String scoreIndexName = createIndex("CREATE CUSTOM INDEX ON %s (score) USING 'StorageAttachedIndex'");
-        String mapIndexName = createIndex("CREATE CUSTOM INDEX ON %s (map_category) USING 'StorageAttachedIndex'");
-        insertCollectionData();
-        int totalTermsCount = IntStream.range(0, DATASET.length)
-                                       .map(this::calculateTotalTermsForRow)
-                                       .sum();
-
-        assertNumRowsMemtable(scoreIndexName, DATASET.length, DATASET.length);
-        assertNumRowsMemtable(bodyIndexName, DATASET.length, totalTermsCount);
-        assertNumRowsMemtable(mapIndexName, DATASET.length);
-        execute("DELETE FROM %s WHERE id = ?", 4);
-        // Deletion is not tracked by Memindex
-        assertNumRowsMemtable(bodyIndexName, DATASET.length, totalTermsCount);
-        // Test an update to different value for analyzed index
-        execute("UPDATE %s SET body = ? WHERE id = ?", DATASET[10][DATASET_BODY_COLUMN], 6);
-        totalTermsCount += calculateTotalTermsForRow(10) - calculateTotalTermsForRow(6);
-        assertNumRowsMemtable(bodyIndexName, DATASET.length, totalTermsCount);
-        // Update back to the original value
-        execute("UPDATE %s SET body = ? WHERE id = ?", DATASET[6][DATASET_BODY_COLUMN], 10);
-        totalTermsCount += calculateTotalTermsForRow(6) - calculateTotalTermsForRow(10);
-        assertNumRowsMemtable(bodyIndexName, DATASET.length, totalTermsCount);
-        // Flush will account for the deleted row
-        totalTermsCount -= calculateTotalTermsForRow(4);
-        flush();
-        assertNumRowsAndTotalTermsSSTable(scoreIndexName, DATASET.length - 1, DATASET.length - 1);
-        assertNumRowsAndTotalTermsSSTable(bodyIndexName, DATASET.length - 1, totalTermsCount);
-        assertNumRowsSSTable(mapIndexName, DATASET.length - 1);
-        execute("DELETE FROM %s WHERE id = ?", 9);
-        flush();
-        assertNumRowsAndTotalTermsSSTable(scoreIndexName, DATASET.length - 1, DATASET.length - 1);
-        assertNumRowsAndTotalTermsSSTable(bodyIndexName, DATASET.length - 1, totalTermsCount);
-        assertNumRowsSSTable(mapIndexName, DATASET.length - 1);
-        compact();
-        totalTermsCount -= calculateTotalTermsForRow(9);
-        assertNumRowsAndTotalTermsSSTable(scoreIndexName, DATASET.length - 2, DATASET.length - 2);
-        assertNumRowsAndTotalTermsSSTable(bodyIndexName, DATASET.length - 2, totalTermsCount);
-        assertNumRowsSSTable(mapIndexName, DATASET.length - 2);
-    }
-
-    private void assertNumRowsMemtable(String indexName, int expectedNumRows)
-    {
-        assertNumRowsMemtable(indexName, expectedNumRows, -1);
-    }
-
-    private void assertNumRowsMemtable(String indexName, int expectedNumRows, int expectedTotalTermsCount)
-    {
-        int rowCount = 0;
-        long termCount = 0;
-
-        for (var memtable : getCurrentColumnFamilyStore().getAllMemtables())
-        {
-            MemtableIndex memIndex = getIndexContext(indexName).getLiveMemtables().get(memtable);
-            assert memIndex instanceof TrieMemtableIndex;
-            rowCount += ((TrieMemtableIndex) memIndex).indexedRows();
-            termCount += ((TrieMemtableIndex) memIndex).approximateTotalTermCount();
-        }
-        assertEquals(expectedNumRows, rowCount);
-        if (expectedTotalTermsCount >= 0)
-            assertEquals(expectedTotalTermsCount, termCount);
-    }
-
-    private void assertNumRowsSSTable(String indexName, int expectedNumRows)
-    {
-        assertNumRowsAndTotalTermsSSTable(indexName, expectedNumRows, -1);
-    }
-
-    private void assertNumRowsAndTotalTermsSSTable(String indexName, int expectedNumRows, int expectedTotalTermsCount
-    )
-    {
-        long indexRowCount = 0;
-        long segmentRowCount = 0;
-        long totalTermCount = 0;
-        for (SSTableIndex sstableIndex : getIndexContext(indexName).getView())
-        {
-            indexRowCount += sstableIndex.getRowCount();
-            for (var segment : sstableIndex.getSegments())
-            {
-                var metadata = segment.metadata;
-                Assert.assertNotNull(metadata);
-                segmentRowCount += metadata.numRows;
-                totalTermCount += metadata.totalTermCount;
-            }
-        }
-        assertEquals(indexRowCount, segmentRowCount);
-        assertEquals(expectedNumRows, indexRowCount);
-        if (expectedTotalTermsCount >= 0)
-            assertEquals(expectedTotalTermsCount, totalTermCount);
-    }
-
-    private final static Object[][] DATASET =
+    public final static Object[][] DATASET =
     {
     { 0, "Climate", 5, "Climate change is a pressing issue. Climate patterns are shifting globally. Scientists study climate data daily.", 1 },
     { 1, "Technology", 3, "Technology is advancing. New technology in AI and robotics is groundbreaking.", 1 },
@@ -986,12 +876,6 @@ private void analyzeDataset(String term)
         }
     }
 
-    private int calculateTotalTermsForRow(int row)
-    {
-        String body = (String) DATASET[row][DATASET_BODY_COLUMN];
-        return PATTERN.split(body.toLowerCase()).length;
-    }
-
     private void insertPrimitiveData()
     {
         insertPrimitiveData(0, DATASET.length);
@@ -1012,7 +896,7 @@ private void insertPrimitiveData(int start, int end)
         }
     }
 
-    private void insertCollectionData()
+    public static void insertCollectionData(SAITester tester)
     {
         int setsize = 1;
         for (int row = 0; row < DATASET.length; row++)
@@ -1032,7 +916,7 @@ private void insertCollectionData()
                 map_text.putIfAbsent((String) DATASET[row - j][1], (String) DATASET[row - j][3]);
             }
 
-            execute(
+            tester.execute(
             "INSERT INTO %s (id, category, score, body, bodyset, map_category, map_body) " +
             "VALUES (?, ?, ?, ?, ?, ?, ?)",
             DATASET[row][0],

test/unit/org/apache/cassandra/index/sai/cql/FeaturesVersionSupportTest.java

@@ -16,9 +16,12 @@
 
 package org.apache.cassandra.index.sai.cql;
 
+
 import java.util.Collection;
 import java.util.stream.Collectors;
+import java.util.stream.IntStream;
 
+import org.junit.Assert;
 import org.junit.Before;
 import org.junit.Test;
 import org.junit.runner.RunWith;
@@ -28,13 +31,18 @@
 import org.apache.cassandra.exceptions.ReadFailureException;
 import org.apache.cassandra.exceptions.RequestFailureReason;
 import org.apache.cassandra.index.sai.SAIUtil;
+import org.apache.cassandra.index.sai.SSTableIndex;
 import org.apache.cassandra.index.sai.disk.format.Version;
+import org.apache.cassandra.index.sai.memory.MemtableIndex;
+import org.apache.cassandra.index.sai.memory.TrieMemtableIndex;
 import org.assertj.core.api.Assertions;
 
+import static org.apache.cassandra.index.sai.cql.BM25Test.*;
 import static org.assertj.core.api.Assertions.assertThat;
+import static org.junit.Assert.assertEquals;
 
 /**
- * Tests the availabilty of features in different versions of the SAI on-disk format.
+ * Tests the availability of features in different versions of the SAI on-disk format.
  */
 @RunWith(Parameterized.class)
 public class FeaturesVersionSupportTest extends VectorTester
@@ -226,4 +234,121 @@ public void testQueryAnalyzer()
         assertRows(execute("SELECT k FROM %s WHERE v : 'fox'"), row(1), row(2), row(3));
         assertRows(execute("SELECT k FROM %s WHERE v : 'foxes'"), row(3));
     }
+
+    /**
+     * Asserts that memtable SAI index maintains expected row count, which is, then,
+     * used to store row count in SSTable SAI index and its segments. This is also
+     * asserted.
+     */
+    @Test
+    public void testIndexMetaForNumRows()
+    {
+        SAIUtil.setCurrentVersion(Version.ED);
+
+        createTable("CREATE TABLE %s (id int PRIMARY KEY, category text, score int, " +
+                    "title text, body text, bodyset set<text>, " +
+                    "map_category map<int, text>, map_body map<text, text>)");
+        String bodyIndexName = createIndex("CREATE CUSTOM INDEX ON %s(body) " +
+                                            "USING 'org.apache.cassandra.index.sai.StorageAttachedIndex' " +
+                                            "WITH OPTIONS = {" +
+                                            "'index_analyzer': '{" +
+                                            "\"tokenizer\" : {\"name\" : \"standard\"}, " +
+                                            "\"filters\" : [{\"name\" : \"porterstem\"}" +
+                                            ", {\"name\" : \"lowercase\"}]" +
+                                            "}'}"
+        );
+        String scoreIndexName = createIndex("CREATE CUSTOM INDEX ON %s (score) USING 'StorageAttachedIndex'");
+        String mapIndexName = createIndex("CREATE CUSTOM INDEX ON %s (map_category) USING 'StorageAttachedIndex'");
+        insertCollectionData(this);
+        int totalTermsCount = IntStream.range(0, DATASET.length)
+                                       .map(this::calculateTotalTermsForRow)
+                                       .sum();
+
+        assertNumRowsMemtable(scoreIndexName, DATASET.length, DATASET.length);
+        assertNumRowsMemtable(bodyIndexName, DATASET.length, totalTermsCount);
+        assertNumRowsMemtable(mapIndexName, DATASET.length);
+        execute("DELETE FROM %s WHERE id = ?", 4);
+        // Deletion is not tracked by Memindex
+        assertNumRowsMemtable(bodyIndexName, DATASET.length, totalTermsCount);
+        // Test an update to a different value for the analyzed index
+        execute("UPDATE %s SET body = ? WHERE id = ?", DATASET[10][DATASET_BODY_COLUMN], 6);
+        totalTermsCount += calculateTotalTermsForRow(10) - calculateTotalTermsForRow(6);
+        assertNumRowsMemtable(bodyIndexName, DATASET.length, totalTermsCount);
+        // Update back to the original value
+        execute("UPDATE %s SET body = ? WHERE id = ?", DATASET[6][DATASET_BODY_COLUMN], 10);
+        totalTermsCount += calculateTotalTermsForRow(6) - calculateTotalTermsForRow(10);
+        assertNumRowsMemtable(bodyIndexName, DATASET.length, totalTermsCount);
+        // Flush will account for the deleted row
+        totalTermsCount -= calculateTotalTermsForRow(4);
+        flush();
+        assertNumRowsAndTotalTermsSSTable(scoreIndexName, DATASET.length - 1, DATASET.length - 1);
+        assertNumRowsAndTotalTermsSSTable(bodyIndexName, DATASET.length - 1, totalTermsCount);
+        assertNumRowsSSTable(mapIndexName, DATASET.length - 1);
+        execute("DELETE FROM %s WHERE id = ?", 9);
+        flush();
+        assertNumRowsAndTotalTermsSSTable(scoreIndexName, DATASET.length - 1, DATASET.length - 1);
+        assertNumRowsAndTotalTermsSSTable(bodyIndexName, DATASET.length - 1, totalTermsCount);
+        assertNumRowsSSTable(mapIndexName, DATASET.length - 1);
+        compact();
+        totalTermsCount -= calculateTotalTermsForRow(9);
+        assertNumRowsAndTotalTermsSSTable(scoreIndexName, DATASET.length - 2, DATASET.length - 2);
+        assertNumRowsAndTotalTermsSSTable(bodyIndexName, DATASET.length - 2, totalTermsCount);
+        assertNumRowsSSTable(mapIndexName, DATASET.length - 2);
+    }
+
+    private int calculateTotalTermsForRow(int row)
+    {
+        String body = (String) DATASET[row][DATASET_BODY_COLUMN];
+        return PATTERN.split(body.toLowerCase()).length;
+    }
+
+    private void assertNumRowsMemtable(String indexName, int expectedNumRows)
+    {
+        assertNumRowsMemtable(indexName, expectedNumRows, -1);
+    }
+
+    private void assertNumRowsMemtable(String indexName, int expectedNumRows, int expectedTotalTermsCount)
+    {
+        int rowCount = 0;
+        long termCount = 0;
+
+        for (var memtable : getCurrentColumnFamilyStore().getAllMemtables())
+        {
+            MemtableIndex memIndex = getIndexContext(indexName).getLiveMemtables().get(memtable);
+            assert memIndex instanceof TrieMemtableIndex;
+            rowCount += ((TrieMemtableIndex) memIndex).indexedRows();
+            termCount += ((TrieMemtableIndex) memIndex).approximateTotalTermCount();
+        }
+        assertEquals(expectedNumRows, rowCount);
+        if (expectedTotalTermsCount >= 0)
+            assertEquals(expectedTotalTermsCount, termCount);
+    }
+
+    private void assertNumRowsSSTable(String indexName, int expectedNumRows)
+    {
+        assertNumRowsAndTotalTermsSSTable(indexName, expectedNumRows, -1);
+    }
+
+    private void assertNumRowsAndTotalTermsSSTable(String indexName, int expectedNumRows, int expectedTotalTermsCount
+    )
+    {
+        long indexRowCount = 0;
+        long segmentRowCount = 0;
+        long totalTermCount = 0;
+        for (SSTableIndex sstableIndex : getIndexContext(indexName).getView())
+        {
+            indexRowCount += sstableIndex.getRowCount();
+            for (var segment : sstableIndex.getSegments())
+            {
+                var metadata = segment.metadata;
+                Assert.assertNotNull(metadata);
+                segmentRowCount += metadata.numRows;
+                totalTermCount += metadata.totalTermCount;
+            }
+        }
+        assertEquals(indexRowCount, segmentRowCount);
+        assertEquals(expectedNumRows, indexRowCount);
+        if (expectedTotalTermsCount >= 0)
+            assertEquals(expectedTotalTermsCount, totalTermCount);
+    }
 }