Skip to content

Commit a0f97a9

Browse files
netudimanetudima
committed
Reduce performance impact of TableMetadataRef.get and KeyspaceMetadataRef.get
Move TableMetadataRef.get to init part of bulk operations (such as flush, compaction, scrab) Cache TableMetadata value within TableMetadataRef to reduce overheads for write operations, same for KeyspaceMetadata Patch by Dmitry Konstantinov; reviewed by Marcus Eriksson for CASSANDRA-20465
1 parent 909ed8b commit a0f97a9

File tree

13 files changed

+157
-27
lines changed

13 files changed

+157
-27
lines changed

‎CHANGES.txt‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
5.1
2+
* Reduce performance impact of TableMetadataRef.get and KeyspaceMetadataRef.get (CASSANDRA-20465)
23
* Improve CMS initialization (CASSANDRA-21036)
34
* Introducing comments and security labels for schema elements (CASSANDRA-20943)
45
* Extend nodetool tablestats for dictionary memory usage (CASSANDRA-20940)

‎src/java/org/apache/cassandra/db/Keyspace.java‎

Lines changed: 39 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@
2525
import java.util.List;
2626
import java.util.Objects;
2727
import java.util.Set;
28+
import java.util.UUID;
2829
import java.util.concurrent.ConcurrentHashMap;
2930
import java.util.concurrent.ConcurrentMap;
3031
import java.util.concurrent.atomic.AtomicLong;
@@ -723,6 +724,20 @@ private static class KeyspaceMetadataRef
723724
private final String name;
724725
private final SchemaProvider provider;
725726

727+
private volatile KeyspaceMetadataCache cachedKeyspaceMetadata;
728+
729+
private static class KeyspaceMetadataCache
730+
{
731+
private final UUID lastSeenSchemaVersion;
732+
private final KeyspaceMetadata keyspaceMetadata;
733+
734+
private KeyspaceMetadataCache(UUID lastSeenSchemaVersion, KeyspaceMetadata keyspaceMetadata)
735+
{
736+
this.lastSeenSchemaVersion = lastSeenSchemaVersion;
737+
this.keyspaceMetadata = keyspaceMetadata;
738+
}
739+
}
740+
726741
public KeyspaceMetadataRef(KeyspaceMetadata initial, SchemaProvider provider)
727742
{
728743
this.initial = initial;
@@ -734,7 +749,30 @@ public KeyspaceMetadata get()
734749
{
735750
if (initial != null)
736751
return initial;
737-
return provider.getKeyspaceMetadata(name);
752+
return getWithCaching();
753+
}
754+
755+
private KeyspaceMetadata getWithCaching()
756+
{
757+
UUID schemaVersion = provider.getVersion();
758+
if (schemaVersion == null)
759+
return provider.getKeyspaceMetadata(name);
760+
761+
KeyspaceMetadataCache cache = cachedKeyspaceMetadata;
762+
// we assume that local keyspaces and virtual keyspaces are immutable, so we need to track only a distributed schema version
763+
KeyspaceMetadata metadata;
764+
if (cache != null && schemaVersion.equals(cache.lastSeenSchemaVersion) && cache.keyspaceMetadata != null)
765+
metadata = cache.keyspaceMetadata;
766+
else
767+
{
768+
// we always retrieve metadata after schema version and assume they are changed coherently
769+
// we may put new metadata + old schema version to the cache but not vice versa
770+
// it we put non-latest schema version + latest metadata then it will be just updated on the next get() invocation
771+
metadata = provider.getKeyspaceMetadata(name);
772+
if (metadata != null)
773+
cachedKeyspaceMetadata = new KeyspaceMetadataCache(schemaVersion, metadata);
774+
}
775+
return metadata;
738776
}
739777

740778
public void unsetInitial()

‎src/java/org/apache/cassandra/db/memtable/ShardedSkipListMemtable.java‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -284,6 +284,7 @@ public FlushablePartitionSet<AtomicBTreePartition> getFlushSet(PartitionPosition
284284
{
285285
long keySize = 0;
286286
int keyCount = 0;
287+
TableMetadata currentTableMetadata = metadata();
287288

288289
for (Iterator<AtomicBTreePartition> it = getPartitionIterator(from, true, to,false); it.hasNext();)
289290
{
@@ -297,6 +298,8 @@ public FlushablePartitionSet<AtomicBTreePartition> getFlushSet(PartitionPosition
297298

298299
return new AbstractFlushablePartitionSet<AtomicBTreePartition>()
299300
{
301+
private final TableMetadata tableMetadata = currentTableMetadata;
302+
300303
public Memtable memtable()
301304
{
302305
return ShardedSkipListMemtable.this;
@@ -326,6 +329,12 @@ public long partitionKeysSize()
326329
{
327330
return partitionKeySize;
328331
}
332+
333+
@Override
334+
public TableMetadata metadata()
335+
{
336+
return tableMetadata;
337+
}
329338
};
330339
}
331340

‎src/java/org/apache/cassandra/db/memtable/SkipListMemtable.java‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -257,6 +257,7 @@ public FlushablePartitionSet<?> getFlushSet(PartitionPosition from, PartitionPos
257257
Map<PartitionPosition, AtomicBTreePartition> toFlush = getPartitionsSubMap(from, true, to, false);
258258
long keysSize = 0;
259259
long keyCount = 0;
260+
TableMetadata currentTableMetadata = metadata();
260261

261262
boolean trackContention = logger.isTraceEnabled();
262263
if (trackContention)
@@ -289,6 +290,8 @@ public FlushablePartitionSet<?> getFlushSet(PartitionPosition from, PartitionPos
289290

290291
return new AbstractFlushablePartitionSet<AtomicBTreePartition>()
291292
{
293+
private final TableMetadata tableMetadata = currentTableMetadata;
294+
292295
@Override
293296
public Memtable memtable()
294297
{
@@ -324,6 +327,12 @@ public long partitionKeysSize()
324327
{
325328
return partitionKeysSize;
326329
}
330+
331+
@Override
332+
public TableMetadata metadata()
333+
{
334+
return tableMetadata;
335+
}
327336
};
328337
}
329338

‎src/java/org/apache/cassandra/db/memtable/TrieMemtable.java‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -448,6 +448,8 @@ public FlushablePartitionSet<MemtablePartition> getFlushSet(PartitionPosition fr
448448

449449
return new AbstractFlushablePartitionSet<MemtablePartition>()
450450
{
451+
private final TableMetadata tableMetadata = TrieMemtable.this.metadata();
452+
451453
public Memtable memtable()
452454
{
453455
return TrieMemtable.this;
@@ -480,6 +482,12 @@ public long partitionKeysSize()
480482
{
481483
return partitionKeySize;
482484
}
485+
486+
@Override
487+
public TableMetadata metadata()
488+
{
489+
return tableMetadata;
490+
}
483491
};
484492
}
485493

‎src/java/org/apache/cassandra/io/sstable/SSTableIdentityIterator.java‎

Lines changed: 26 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -61,14 +61,19 @@ public SSTableIdentityIterator(SSTableReader sstable, DecoratedKey key, Deletion
6161
}
6262

6363
public static SSTableIdentityIterator create(SSTableReader sstable, RandomAccessReader file, DecoratedKey key)
64+
{
65+
return create(sstable, sstable.metadata(), file, key);
66+
}
67+
68+
public static SSTableIdentityIterator create(SSTableReader sstable, TableMetadata tableMetadata, RandomAccessReader file, DecoratedKey key)
6469
{
6570
try
6671
{
6772
DeletionTime partitionLevelDeletion = DeletionTime.getSerializer(sstable.descriptor.version).deserialize(file);
6873
if (!partitionLevelDeletion.validate())
69-
UnfilteredValidation.handleInvalid(sstable.metadata(), key, sstable, "partitionLevelDeletion="+partitionLevelDeletion.toString());
70-
DeserializationHelper helper = new DeserializationHelper(sstable.metadata(), sstable.descriptor.version.correspondingMessagingVersion(), DeserializationHelper.Flag.LOCAL);
71-
SSTableSimpleIterator iterator = SSTableSimpleIterator.create(sstable.metadata(), file, sstable.header, helper, partitionLevelDeletion);
74+
UnfilteredValidation.handleInvalid(tableMetadata, key, sstable, "partitionLevelDeletion="+partitionLevelDeletion.toString());
75+
DeserializationHelper helper = new DeserializationHelper(tableMetadata, sstable.descriptor.version.correspondingMessagingVersion(), DeserializationHelper.Flag.LOCAL);
76+
SSTableSimpleIterator iterator = SSTableSimpleIterator.create(tableMetadata, file, sstable.header, helper, partitionLevelDeletion);
7277
return new SSTableIdentityIterator(sstable, key, partitionLevelDeletion, file.getPath(), iterator);
7378
}
7479
catch (IOException e)
@@ -84,19 +89,24 @@ public static SSTableIdentityIterator create(SSTableReader sstable, RandomAccess
8489
}
8590

8691
public static SSTableIdentityIterator create(SSTableReader sstable, FileDataInput dfile, long dataPosition, DecoratedKey key, boolean tombstoneOnly)
92+
{
93+
return create(sstable, sstable.metadata(), dfile, dataPosition, key, tombstoneOnly);
94+
}
95+
96+
public static SSTableIdentityIterator create(SSTableReader sstable, TableMetadata tableMetadata, FileDataInput dfile, long dataPosition, DecoratedKey key, boolean tombstoneOnly)
8797
{
8898
try
8999
{
90100
dfile.seek(dataPosition);
91101
ByteBufferUtil.skipShortLength(dfile); // Skip partition key
92102
DeletionTime partitionLevelDeletion = DeletionTime.getSerializer(sstable.descriptor.version).deserialize(dfile);
93103
if (!partitionLevelDeletion.validate())
94-
UnfilteredValidation.handleInvalid(sstable.metadata(), key, sstable, "partitionLevelDeletion="+partitionLevelDeletion.toString());
104+
UnfilteredValidation.handleInvalid(tableMetadata, key, sstable, "partitionLevelDeletion="+partitionLevelDeletion.toString());
95105

96-
DeserializationHelper helper = new DeserializationHelper(sstable.metadata(), sstable.descriptor.version.correspondingMessagingVersion(), DeserializationHelper.Flag.LOCAL);
106+
DeserializationHelper helper = new DeserializationHelper(tableMetadata, sstable.descriptor.version.correspondingMessagingVersion(), DeserializationHelper.Flag.LOCAL);
97107
SSTableSimpleIterator iterator = tombstoneOnly
98-
? SSTableSimpleIterator.createTombstoneOnly(sstable.metadata(), dfile, sstable.header, helper, partitionLevelDeletion)
99-
: SSTableSimpleIterator.create(sstable.metadata(), dfile, sstable.header, helper, partitionLevelDeletion);
108+
? SSTableSimpleIterator.createTombstoneOnly(tableMetadata, dfile, sstable.header, helper, partitionLevelDeletion)
109+
: SSTableSimpleIterator.create(tableMetadata, dfile, sstable.header, helper, partitionLevelDeletion);
100110
return new SSTableIdentityIterator(sstable, key, partitionLevelDeletion, dfile.getPath(), iterator);
101111
}
102112
catch (IOException e)
@@ -112,18 +122,23 @@ public static SSTableIdentityIterator create(SSTableReader sstable, FileDataInpu
112122
}
113123

114124
public static SSTableIdentityIterator create(SSTableReader sstable, FileDataInput dfile, boolean tombstoneOnly)
125+
{
126+
return create(sstable, sstable.metadata(), dfile, tombstoneOnly);
127+
}
128+
129+
public static SSTableIdentityIterator create(SSTableReader sstable, TableMetadata tableMetadata, FileDataInput dfile, boolean tombstoneOnly)
115130
{
116131
try
117132
{
118133
DecoratedKey key = sstable.decorateKey(ByteBufferUtil.readWithShortLength(dfile));
119134
DeletionTime partitionLevelDeletion = DeletionTime.getSerializer(sstable.descriptor.version).deserialize(dfile);
120135
if (!partitionLevelDeletion.validate())
121-
UnfilteredValidation.handleInvalid(sstable.metadata(), key, sstable, "partitionLevelDeletion="+partitionLevelDeletion.toString());
136+
UnfilteredValidation.handleInvalid(tableMetadata, key, sstable, "partitionLevelDeletion="+partitionLevelDeletion.toString());
122137

123-
DeserializationHelper helper = new DeserializationHelper(sstable.metadata(), sstable.descriptor.version.correspondingMessagingVersion(), DeserializationHelper.Flag.LOCAL);
138+
DeserializationHelper helper = new DeserializationHelper(tableMetadata, sstable.descriptor.version.correspondingMessagingVersion(), DeserializationHelper.Flag.LOCAL);
124139
SSTableSimpleIterator iterator = tombstoneOnly
125-
? SSTableSimpleIterator.createTombstoneOnly(sstable.metadata(), dfile, sstable.header, helper, partitionLevelDeletion)
126-
: SSTableSimpleIterator.create(sstable.metadata(), dfile, sstable.header, helper, partitionLevelDeletion);
140+
? SSTableSimpleIterator.createTombstoneOnly(tableMetadata, dfile, sstable.header, helper, partitionLevelDeletion)
141+
: SSTableSimpleIterator.create(tableMetadata, dfile, sstable.header, helper, partitionLevelDeletion);
127142
return new SSTableIdentityIterator(sstable, key, partitionLevelDeletion, dfile.getPath(), iterator);
128143
}
129144
catch (IOException e)

‎src/java/org/apache/cassandra/io/sstable/format/SSTableSimpleScanner.java‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -48,6 +48,8 @@ public class SSTableSimpleScanner
4848
private final RandomAccessReader dfile;
4949
private final SSTableReader sstable;
5050

51+
private final TableMetadata tableMetadata;
52+
5153
private final Iterator<PartitionPositionBounds> rangeIterator;
5254

5355
private long bytesScannedInPreviousRanges;
@@ -75,6 +77,7 @@ public SSTableSimpleScanner(SSTableReader sstable,
7577

7678
this.dfile = sstable.openDataReaderForScan();
7779
this.sstable = sstable;
80+
this.tableMetadata = sstable.metadata();
7881
this.sizeInBytes = boundsList.stream().mapToLong(ppb -> ppb.upperPosition - ppb.lowerPosition).sum();
7982
this.compressedSizeInBytes = sstable.compression ? sstable.onDiskSizeForPartitionPositions(boundsList) : sizeInBytes;
8083
this.rangeIterator = boundsList.iterator();
@@ -190,7 +193,7 @@ public UnfilteredRowIterator next()
190193
if (!hasNext())
191194
throw new NoSuchElementException();
192195

193-
currentIterator = SSTableIdentityIterator.create(sstable, dfile, false);
196+
currentIterator = SSTableIdentityIterator.create(sstable, tableMetadata, dfile, false);
194197
DecoratedKey currentKey = currentIterator.partitionKey();
195198
if (lastKey != null && lastKey.compareTo(currentKey) >= 0)
196199
{

‎src/java/org/apache/cassandra/io/sstable/format/big/BigSSTableReaderLoadingBuilder.java‎

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -46,6 +46,7 @@
4646
import org.apache.cassandra.io.util.FileHandle;
4747
import org.apache.cassandra.io.util.RandomAccessReader;
4848
import org.apache.cassandra.metrics.TableMetrics;
49+
import org.apache.cassandra.schema.TableMetadata;
4950
import org.apache.cassandra.service.CacheService;
5051
import org.apache.cassandra.utils.ByteBufferUtil;
5152
import org.apache.cassandra.utils.FilterFactory;
@@ -199,22 +200,23 @@ private Pair<IFilter, IndexSummaryComponent> buildSummaryAndBloomFilter(FileHand
199200
DecoratedKey first = null, key = null;
200201
IFilter bf = null;
201202
IndexSummary indexSummary = null;
203+
TableMetadata tableMetadata = tableMetadataRef.getLocal();
202204

203205
// we read the positions in a BRAF, so we don't have to worry about an entry spanning a mmap boundary.
204206
try (KeyReader keyReader = createKeyReader(indexFile, serializationHeader, tableMetrics))
205207
{
206208
long estimatedRowsNumber = rebuildFilter || rebuildSummary ? estimateRowsFromIndex(indexFile) : 0;
207209

208210
if (rebuildFilter)
209-
bf = FilterFactory.getFilter(estimatedRowsNumber, tableMetadataRef.getLocal().params.bloomFilterFpChance);
211+
bf = FilterFactory.getFilter(estimatedRowsNumber, tableMetadata.params.bloomFilterFpChance);
210212

211213
try (IndexSummaryBuilder summaryBuilder = !rebuildSummary ? null : new IndexSummaryBuilder(estimatedRowsNumber,
212-
tableMetadataRef.getLocal().params.minIndexInterval,
214+
tableMetadata.params.minIndexInterval,
213215
Downsampling.BASE_SAMPLING_LEVEL))
214216
{
215217
while (!keyReader.isExhausted())
216218
{
217-
key = tableMetadataRef.getLocal().partitioner.decorateKey(keyReader.key());
219+
key = tableMetadata.partitioner.decorateKey(keyReader.key());
218220
if (rebuildSummary)
219221
{
220222
if (first == null)
@@ -229,7 +231,7 @@ private Pair<IFilter, IndexSummaryComponent> buildSummaryAndBloomFilter(FileHand
229231
}
230232

231233
if (rebuildSummary)
232-
indexSummary = summaryBuilder.build(tableMetadataRef.getLocal().partitioner);
234+
indexSummary = summaryBuilder.build(tableMetadata.partitioner);
233235
}
234236
}
235237
catch (IOException | RuntimeException | Error ex)

‎src/java/org/apache/cassandra/io/sstable/format/big/BigTableScrubber.java‎

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,7 @@
3333
import org.apache.cassandra.io.sstable.format.big.BigFormat.Components;
3434
import org.apache.cassandra.io.util.FileUtils;
3535
import org.apache.cassandra.io.util.RandomAccessReader;
36+
import org.apache.cassandra.schema.TableMetadata;
3637
import org.apache.cassandra.utils.ByteBufferUtil;
3738
import org.apache.cassandra.utils.FBUtilities;
3839
import org.apache.cassandra.utils.JVMStabilityInspector;
@@ -105,6 +106,7 @@ protected void scrubInternal(SSTableRewriter writer) throws IOException
105106

106107
DecoratedKey prevKey = null;
107108

109+
TableMetadata tableMetadata = cfs.metadata.getLocal();
108110
while (!dataFile.isEOF())
109111
{
110112
if (scrubInfo.isStopRequested())
@@ -117,8 +119,8 @@ protected void scrubInternal(SSTableRewriter writer) throws IOException
117119
try
118120
{
119121
ByteBuffer raw = ByteBufferUtil.readWithShortLength(dataFile);
120-
if (!cfs.metadata.getLocal().isIndex())
121-
cfs.metadata.getLocal().partitionKeyType.validate(raw);
122+
if (!tableMetadata.isIndex())
123+
tableMetadata.partitionKeyType.validate(raw);
122124
key = sstable.decorateKey(raw);
123125
}
124126
catch (Throwable th)
@@ -181,8 +183,8 @@ protected void scrubInternal(SSTableRewriter writer) throws IOException
181183
key = sstable.decorateKey(currentIndexKey);
182184
try
183185
{
184-
if (!cfs.metadata.getLocal().isIndex())
185-
cfs.metadata.getLocal().partitionKeyType.validate(key.getKey());
186+
if (!tableMetadata.isIndex())
187+
tableMetadata.partitionKeyType.validate(key.getKey());
186188
dataFile.seek(dataStartFromIndex);
187189

188190
if (tryAppend(prevKey, key, writer))

‎src/java/org/apache/cassandra/io/sstable/format/bti/BtiTableReaderLoadingBuilder.java‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -41,6 +41,7 @@
4141
import org.apache.cassandra.io.util.FileHandle;
4242
import org.apache.cassandra.metrics.TableMetrics;
4343
import org.apache.cassandra.schema.Schema;
44+
import org.apache.cassandra.schema.TableMetadata;
4445
import org.apache.cassandra.utils.FilterFactory;
4546
import org.apache.cassandra.utils.IFilter;
4647
import org.apache.cassandra.utils.Throwables;
@@ -163,11 +164,12 @@ private IFilter buildBloomFilter(StatsMetadata statsMetadata) throws IOException
163164

164165
try (KeyReader keyReader = createKeyReader(statsMetadata))
165166
{
166-
bf = FilterFactory.getFilter(statsMetadata.totalRows, tableMetadataRef.getLocal().params.bloomFilterFpChance);
167+
TableMetadata tableMetadata = tableMetadataRef.getLocal();
168+
bf = FilterFactory.getFilter(statsMetadata.totalRows, tableMetadata.params.bloomFilterFpChance);
167169

168170
while (!keyReader.isExhausted())
169171
{
170-
DecoratedKey key = tableMetadataRef.getLocal().partitioner.decorateKey(keyReader.key());
172+
DecoratedKey key = tableMetadata.partitioner.decorateKey(keyReader.key());
171173
bf.add(key);
172174

173175
keyReader.advance();

0 commit comments

Comments
 (0)