Skip to content

Improve estimating live docs bytes #132523

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 1 commit into from
Aug 8, 2025
Merged

Improve estimating live docs bytes #132523

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
22 changes: 14 additions & 8 deletions server/src/main/java/org/elasticsearch/index/engine/Engine.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,7 @@
import org.apache.lucene.store.AlreadyClosedException;
import org.apache.lucene.util.Bits;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.FixedBitSet;
import org.apache.lucene.util.RamUsageEstimator;
import org.apache.lucene.util.SetOnce;
import org.elasticsearch.ExceptionsHelper;
Expand Down Expand Up @@ -283,7 +284,7 @@ protected static ShardFieldStats shardFieldStats(List<LeafReaderContext> leaves)
int totalFields = 0;
long usages = 0;
long totalPostingBytes = 0;
long liveDocsBytes = 0;
long totalLiveDocsBytes = 0;
for (LeafReaderContext leaf : leaves) {
numSegments++;
var fieldInfos = leaf.reader().getFieldInfos();
Expand Down Expand Up @@ -312,18 +313,23 @@ protected static ShardFieldStats shardFieldStats(List<LeafReaderContext> leaves)
var liveDocs = segmentReader.getLiveDocs();
if (liveDocs != null) {
assert validateLiveDocsClass(liveDocs);
// Would prefer to use FixedBitSet#ramBytesUsed() however FixedBits / Bits interface don't expose that.
// This almost does what FixedBitSet#ramBytesUsed() does, liveDocs.length() returns the length of the bits long
// array
liveDocsBytes += RamUsageEstimator.alignObjectSize(
(long) RamUsageEstimator.NUM_BYTES_ARRAY_HEADER + (liveDocs.length() / 8L)
);
long liveDocsBytes = getLiveDocsBytes(liveDocs);
totalLiveDocsBytes += liveDocsBytes;
}
}
}
}
}
return new ShardFieldStats(numSegments, totalFields, usages, totalPostingBytes, liveDocsBytes);
return new ShardFieldStats(numSegments, totalFields, usages, totalPostingBytes, totalLiveDocsBytes);
}

// Would prefer to use FixedBitSet#ramBytesUsed() however FixedBits / Bits interface don't expose that.
// This simulates FixedBitSet#ramBytesUsed() does:
private static long getLiveDocsBytes(Bits liveDocs) {
int words = FixedBitSet.bits2words(liveDocs.length());
return ShardFieldStats.FIXED_BITSET_BASE_RAM_BYTES_USED + RamUsageEstimator.alignObjectSize(
RamUsageEstimator.sizeOf(RamUsageEstimator.NUM_BYTES_ARRAY_HEADER + (long) Long.BYTES * words)
);
}

private static boolean validateLiveDocsClass(Bits liveDocs) {
Expand Down
3 changes: 3 additions & 0 deletions server/src/main/java/org/elasticsearch/index/shard/ShardFieldStats.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,8 @@

package org.elasticsearch.index.shard;

import org.apache.lucene.util.FixedBitSet;
import org.apache.lucene.util.RamUsageEstimator;
import org.elasticsearch.common.util.FeatureFlag;

/**
Expand All @@ -25,5 +27,6 @@
public record ShardFieldStats(int numSegments, int totalFields, long fieldUsages, long postingsInMemoryBytes, long liveDocsBytes) {

public static final FeatureFlag TRACK_LIVE_DOCS_IN_MEMORY_BYTES = new FeatureFlag("track_live_docs_in_memory_bytes");
public static final long FIXED_BITSET_BASE_RAM_BYTES_USED = RamUsageEstimator.shallowSizeOfInstance(FixedBitSet.class);

}
29 changes: 24 additions & 5 deletions server/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@
import org.apache.lucene.store.IOContext;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.Constants;
import org.apache.lucene.util.FixedBitSet;
import org.elasticsearch.ElasticsearchException;
import org.elasticsearch.ExceptionsHelper;
import org.elasticsearch.action.ActionListener;
Expand Down Expand Up @@ -77,6 +78,7 @@
import org.elasticsearch.index.IndexModule;
import org.elasticsearch.index.IndexSettings;
import org.elasticsearch.index.IndexVersion;
import org.elasticsearch.index.MergePolicyConfig;
import org.elasticsearch.index.codec.CodecService;
import org.elasticsearch.index.codec.TrackingPostingsInMemoryBytesCodec;
import org.elasticsearch.index.engine.CommitStats;
Expand Down Expand Up @@ -1981,7 +1983,10 @@ public void testShardFieldStats() throws IOException {
}

public void testShardFieldStatsWithDeletes() throws IOException {
Settings settings = Settings.builder().put(IndexSettings.INDEX_REFRESH_INTERVAL_SETTING.getKey(), TimeValue.MINUS_ONE).build();
Settings settings = Settings.builder()
.put(MergePolicyConfig.INDEX_MERGE_ENABLED, false)
.put(IndexSettings.INDEX_REFRESH_INTERVAL_SETTING.getKey(), TimeValue.MINUS_ONE)
.build();
IndexShard shard = newShard(true, settings);
assertNull(shard.getShardFieldStats());
recoverShardFromStore(shard);
Expand Down Expand Up @@ -2010,8 +2015,14 @@ public void testShardFieldStatsWithDeletes() throws IOException {
stats = shard.getShardFieldStats();
// More segments because delete operation is stored in the new segment for replication purposes.
assertThat(stats.numSegments(), equalTo(2));
// Delete op is stored in new segment, but marked as deleted. All segements have live docs:
assertThat(stats.liveDocsBytes(), equalTo(liveDocsTrackingEnabled ? 40L : 0L));
long expectedLiveDocsSize = 0;
if (liveDocsTrackingEnabled) {
// Delete op is stored in new segment, but marked as deleted. All segements have live docs:
expectedLiveDocsSize += new FixedBitSet(numDocs).ramBytesUsed();
// Second segment the delete operation that is marked as deleted:
expectedLiveDocsSize += new FixedBitSet(1).ramBytesUsed();
}
assertThat(stats.liveDocsBytes(), equalTo(expectedLiveDocsSize));

// delete another doc:
deleteDoc(shard, "first_1");
Expand All @@ -2022,8 +2033,16 @@ public void testShardFieldStatsWithDeletes() throws IOException {
stats = shard.getShardFieldStats();
// More segments because delete operation is stored in the new segment for replication purposes.
assertThat(stats.numSegments(), equalTo(3));
// Delete op is stored in new segment, but marked as deleted. All segements have live docs:
assertThat(stats.liveDocsBytes(), equalTo(liveDocsTrackingEnabled ? 56L : 0L));
expectedLiveDocsSize = 0;
if (liveDocsTrackingEnabled) {
// Delete op is stored in new segment, but marked as deleted. All segements have live docs:
// First segment with deletes
expectedLiveDocsSize += new FixedBitSet(numDocs).ramBytesUsed();
// Second and third segments the delete operation that is marked as deleted:
expectedLiveDocsSize += new FixedBitSet(1).ramBytesUsed();
expectedLiveDocsSize += new FixedBitSet(1).ramBytesUsed();
}
assertThat(stats.liveDocsBytes(), equalTo(expectedLiveDocsSize));

closeShards(shard);
}
Expand Down