diff --git a/server/src/main/java/org/elasticsearch/index/engine/Engine.java b/server/src/main/java/org/elasticsearch/index/engine/Engine.java index 88c36e2061f84..20498a0dffe71 100644 --- a/server/src/main/java/org/elasticsearch/index/engine/Engine.java +++ b/server/src/main/java/org/elasticsearch/index/engine/Engine.java @@ -34,6 +34,7 @@ import org.apache.lucene.store.AlreadyClosedException; import org.apache.lucene.util.Bits; import org.apache.lucene.util.BytesRef; +import org.apache.lucene.util.FixedBitSet; import org.apache.lucene.util.RamUsageEstimator; import org.apache.lucene.util.SetOnce; import org.elasticsearch.ExceptionsHelper; @@ -283,7 +284,7 @@ protected static ShardFieldStats shardFieldStats(List leaves) int totalFields = 0; long usages = 0; long totalPostingBytes = 0; - long liveDocsBytes = 0; + long totalLiveDocsBytes = 0; for (LeafReaderContext leaf : leaves) { numSegments++; var fieldInfos = leaf.reader().getFieldInfos(); @@ -312,18 +313,23 @@ protected static ShardFieldStats shardFieldStats(List leaves) var liveDocs = segmentReader.getLiveDocs(); if (liveDocs != null) { assert validateLiveDocsClass(liveDocs); - // Would prefer to use FixedBitSet#ramBytesUsed() however FixedBits / Bits interface don't expose that. - // This almost does what FixedBitSet#ramBytesUsed() does, liveDocs.length() returns the length of the bits long - // array - liveDocsBytes += RamUsageEstimator.alignObjectSize( - (long) RamUsageEstimator.NUM_BYTES_ARRAY_HEADER + (liveDocs.length() / 8L) - ); + long liveDocsBytes = getLiveDocsBytes(liveDocs); + totalLiveDocsBytes += liveDocsBytes; } } } } } - return new ShardFieldStats(numSegments, totalFields, usages, totalPostingBytes, liveDocsBytes); + return new ShardFieldStats(numSegments, totalFields, usages, totalPostingBytes, totalLiveDocsBytes); + } + + // Would prefer to use FixedBitSet#ramBytesUsed() however FixedBits / Bits interface don't expose that. + // This simulates FixedBitSet#ramBytesUsed() does: + private static long getLiveDocsBytes(Bits liveDocs) { + int words = FixedBitSet.bits2words(liveDocs.length()); + return ShardFieldStats.FIXED_BITSET_BASE_RAM_BYTES_USED + RamUsageEstimator.alignObjectSize( + RamUsageEstimator.sizeOf(RamUsageEstimator.NUM_BYTES_ARRAY_HEADER + (long) Long.BYTES * words) + ); } private static boolean validateLiveDocsClass(Bits liveDocs) { diff --git a/server/src/main/java/org/elasticsearch/index/shard/ShardFieldStats.java b/server/src/main/java/org/elasticsearch/index/shard/ShardFieldStats.java index 2b2c22227836b..17f9c7240f979 100644 --- a/server/src/main/java/org/elasticsearch/index/shard/ShardFieldStats.java +++ b/server/src/main/java/org/elasticsearch/index/shard/ShardFieldStats.java @@ -9,6 +9,8 @@ package org.elasticsearch.index.shard; +import org.apache.lucene.util.FixedBitSet; +import org.apache.lucene.util.RamUsageEstimator; import org.elasticsearch.common.util.FeatureFlag; /** @@ -25,5 +27,6 @@ public record ShardFieldStats(int numSegments, int totalFields, long fieldUsages, long postingsInMemoryBytes, long liveDocsBytes) { public static final FeatureFlag TRACK_LIVE_DOCS_IN_MEMORY_BYTES = new FeatureFlag("track_live_docs_in_memory_bytes"); + public static final long FIXED_BITSET_BASE_RAM_BYTES_USED = RamUsageEstimator.shallowSizeOfInstance(FixedBitSet.class); } diff --git a/server/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java b/server/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java index 5f4230f83383a..2ff76bc1d6c9c 100644 --- a/server/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java +++ b/server/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java @@ -27,6 +27,7 @@ import org.apache.lucene.store.IOContext; import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.Constants; +import org.apache.lucene.util.FixedBitSet; import org.elasticsearch.ElasticsearchException; import org.elasticsearch.ExceptionsHelper; import org.elasticsearch.action.ActionListener; @@ -77,6 +78,7 @@ import org.elasticsearch.index.IndexModule; import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.IndexVersion; +import org.elasticsearch.index.MergePolicyConfig; import org.elasticsearch.index.codec.CodecService; import org.elasticsearch.index.codec.TrackingPostingsInMemoryBytesCodec; import org.elasticsearch.index.engine.CommitStats; @@ -1981,7 +1983,10 @@ public void testShardFieldStats() throws IOException { } public void testShardFieldStatsWithDeletes() throws IOException { - Settings settings = Settings.builder().put(IndexSettings.INDEX_REFRESH_INTERVAL_SETTING.getKey(), TimeValue.MINUS_ONE).build(); + Settings settings = Settings.builder() + .put(MergePolicyConfig.INDEX_MERGE_ENABLED, false) + .put(IndexSettings.INDEX_REFRESH_INTERVAL_SETTING.getKey(), TimeValue.MINUS_ONE) + .build(); IndexShard shard = newShard(true, settings); assertNull(shard.getShardFieldStats()); recoverShardFromStore(shard); @@ -2010,8 +2015,14 @@ public void testShardFieldStatsWithDeletes() throws IOException { stats = shard.getShardFieldStats(); // More segments because delete operation is stored in the new segment for replication purposes. assertThat(stats.numSegments(), equalTo(2)); - // Delete op is stored in new segment, but marked as deleted. All segements have live docs: - assertThat(stats.liveDocsBytes(), equalTo(liveDocsTrackingEnabled ? 40L : 0L)); + long expectedLiveDocsSize = 0; + if (liveDocsTrackingEnabled) { + // Delete op is stored in new segment, but marked as deleted. All segements have live docs: + expectedLiveDocsSize += new FixedBitSet(numDocs).ramBytesUsed(); + // Second segment the delete operation that is marked as deleted: + expectedLiveDocsSize += new FixedBitSet(1).ramBytesUsed(); + } + assertThat(stats.liveDocsBytes(), equalTo(expectedLiveDocsSize)); // delete another doc: deleteDoc(shard, "first_1"); @@ -2022,8 +2033,16 @@ public void testShardFieldStatsWithDeletes() throws IOException { stats = shard.getShardFieldStats(); // More segments because delete operation is stored in the new segment for replication purposes. assertThat(stats.numSegments(), equalTo(3)); - // Delete op is stored in new segment, but marked as deleted. All segements have live docs: - assertThat(stats.liveDocsBytes(), equalTo(liveDocsTrackingEnabled ? 56L : 0L)); + expectedLiveDocsSize = 0; + if (liveDocsTrackingEnabled) { + // Delete op is stored in new segment, but marked as deleted. All segements have live docs: + // First segment with deletes + expectedLiveDocsSize += new FixedBitSet(numDocs).ramBytesUsed(); + // Second and third segments the delete operation that is marked as deleted: + expectedLiveDocsSize += new FixedBitSet(1).ramBytesUsed(); + expectedLiveDocsSize += new FixedBitSet(1).ramBytesUsed(); + } + assertThat(stats.liveDocsBytes(), equalTo(expectedLiveDocsSize)); closeShards(shard); }