Merge branch 'cassandra-5.0' into trunk

* cassandra-5.0:
  ninja-fix – Fix eclipse-warnings error for CASSANDRA-19564
  ReadCommandController should close fast to avoid deadlock when building secondary index

Author: michaelsembwever

CHANGES.txt

@@ -309,6 +309,7 @@ Merged from 5.0:
  * Prioritize built indexes in IndexStatusManager (CASSANDRA-19400)
  * Add java.base/java.lang.reflect among opens for jvm11-client.options (CASSANDRA-19780)
 Merged from 4.1:
+ * ReadCommandController should close fast to avoid deadlock when building secondary index (CASSANDRA-19564)
  * Redact security-sensitive information in system_views.settings (CASSANDRA-20856)
  * Improve CommitLogSegmentReader to skip SyncBlocks correctly in case of CRC errors (CASSANDRA-20664)
  * Do not crash on first boot with data_disk_usage_max_disk_size set when data directory is not created yet (CASSANDRA-20787)

src/java/org/apache/cassandra/index/SecondaryIndexManager.java

@@ -61,6 +61,7 @@
 import org.apache.cassandra.db.lifecycle.SSTableSet;
 import org.apache.cassandra.db.lifecycle.View;
 import org.apache.cassandra.db.memtable.Memtable;
+import org.apache.cassandra.db.partitions.ImmutableBTreePartition;
 import org.apache.cassandra.db.partitions.PartitionUpdate;
 import org.apache.cassandra.db.partitions.UnfilteredPartitionIterator;
 import org.apache.cassandra.db.rows.*;
@@ -1064,14 +1065,22 @@ public void indexPartition(DecoratedKey key, Set<Index> indexes, int pageSize, R
             SinglePartitionPager pager = new SinglePartitionPager(cmd, null, ProtocolVersion.CURRENT);
             while (!pager.isExhausted())
             {
+                @SuppressWarnings("resource")
+                UnfilteredRowIterator partition;
                 try (ReadExecutionController controller = cmd.executionController();
-                     WriteContext ctx = keyspace.getWriteHandler().createContextForIndexing();
                      UnfilteredPartitionIterator page = pager.fetchPageUnfiltered(baseCfs.metadata(), pageSize, controller))
                 {
                     if (!page.hasNext())
                         break;
 
-                    try (UnfilteredRowIterator partition = page.next())
+                    try (UnfilteredRowIterator onePartition = page.next())
+                    {
+                        partition = ImmutableBTreePartition.create(onePartition).unfilteredIterator();
+                    }
+                }
+
+                try (WriteContext ctx = keyspace.getWriteHandler().createContextForIndexing())
+                {
                     {
                         Set<Index.Indexer> indexers = new HashSet<>(indexGroups.size());
 
@@ -1135,6 +1144,13 @@ public void indexPartition(DecoratedKey key, Set<Index> indexes, int pageSize, R
                         indexers.forEach(Index.Indexer::finish);
                     }
                 }
+                finally
+                {
+                    if (partition != null)
+                    {
+                        partition.close();
+                    }
+                }
             }
         }
     }

test/distributed/org/apache/cassandra/distributed/test/SecondaryIndexTest.java

@@ -23,29 +23,40 @@
 import java.util.Arrays;
 import java.util.Collections;
 import java.util.List;
+import java.util.Random;
 import java.util.UUID;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.regex.Pattern;
 import java.util.stream.Collectors;
 
+import com.google.common.collect.Sets;
+
+import org.awaitility.Awaitility;
 import org.junit.After;
 import org.junit.AfterClass;
 import org.junit.Assert;
 import org.junit.Before;
 import org.junit.BeforeClass;
 import org.junit.Test;
 
+import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.db.Keyspace;
+import org.apache.cassandra.db.marshal.ValueGenerator;
 import org.apache.cassandra.distributed.Cluster;
 import org.apache.cassandra.distributed.api.ConsistencyLevel;
 import org.apache.cassandra.utils.TimeUUID;
-import org.awaitility.Awaitility;
+
+import static org.apache.cassandra.distributed.impl.IsolatedExecutor.waitOn;
 
 public class SecondaryIndexTest extends TestBaseImpl
 {
     private static final int NUM_NODES = 3;
     private static final int REPLICATION_FACTOR = 1;
-    private static final String CREATE_TABLE = "CREATE TABLE %s(k int, v int, PRIMARY KEY (k))";
+    private static final String CREATE_TABLE = "CREATE TABLE %s(k int, v text, PRIMARY KEY (k))";
     private static final String CREATE_INDEX = "CREATE INDEX v_index_%d ON %s(v)";
 
     private static final AtomicInteger seq = new AtomicInteger();
@@ -122,4 +133,48 @@ public void test_only_coordinator_chooses_index_for_query()
                                    });
         }
     }
+
+    @Test
+    public void test_secondary_rebuild_with_small_memtable_memory()
+    {
+        // populate data
+        Random rand = new Random();
+        for (int i = 0 ; i < 100 ; ++i)
+            cluster.coordinator(1).execute(String.format("INSERT INTO %s (k, v) VALUES (?, ?)", tableName), ConsistencyLevel.ALL, i, ValueGenerator.randomString(rand, 50000));
+
+        cluster.forEach(i -> i.flush(KEYSPACE));
+
+        // restart node 1 with small memtable allocation so that index rebuild will cause memtable flush which will need
+        // to reclaim the memory. see CASSANDRA-19564
+        waitOn(cluster.get(1).shutdown());
+        Object originalMemTableHeapSpace = cluster.get(1).config().get("memtable_heap_space");
+        cluster.get(1).config().set("memtable_heap_space", "1MiB");
+        cluster.get(1).startup();
+        String tableNameWithoutKeyspaceName = tableName.split("\\.")[1];
+        String indexName = String.format("v_index_%d", seq.get());
+        Runnable task = cluster.get(1).runsOnInstance(
+        () -> {
+            ColumnFamilyStore cfs = Keyspace.open(KEYSPACE).getColumnFamilyStore(tableNameWithoutKeyspaceName);
+            cfs.indexManager.rebuildIndexesBlocking(Sets.newHashSet(Arrays.asList(indexName)));
+        }
+        );
+        ExecutorService es = Executors.newFixedThreadPool(1);
+        Future<?> future = es.submit(task);
+        try
+        {
+            future.get(30, TimeUnit.SECONDS);
+        }
+        catch (Exception e)
+        {
+            e.printStackTrace();
+            Assert.fail("Rebuild should finish within 30 seconds without issue.");
+        }
+        finally
+        {
+            // restore node1 to use default value for memtable_heap_space
+            waitOn(cluster.get(1).shutdown());
+            cluster.get(1).config().set("memtable_heap_space", originalMemTableHeapSpace);
+            cluster.get(1).startup();
+        }
+    }
 }

test/unit/org/apache/cassandra/db/marshal/ValueGenerator.java

@@ -47,7 +47,12 @@ public static ByteBuffer randomBytes(Random random)
 
     public static String randomString(Random random)
     {
-        char[] chars = new char[random.nextInt(100)];
+        return randomString(random, 100);
+    }
+
+    public static String randomString(Random random, int length)
+    {
+        char[] chars = new char[random.nextInt(length)];
         for (int i=0; i<chars.length; i++)
             chars[i] = CHARS[random.nextInt(CHARS.length)];
         return new String(chars);