Improve estimating live docs bytes (elastic#132523)

By simulating what `FixedBitSet` exactly does.
This also improves testing because we can compare against `FixedBitSet#ramBytesUsed()`.

Follow up from elastic#132232

Author: martijnvg

server/src/main/java/org/elasticsearch/index/engine/Engine.java

@@ -34,6 +34,7 @@
 import org.apache.lucene.store.AlreadyClosedException;
 import org.apache.lucene.util.Bits;
 import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.FixedBitSet;
 import org.apache.lucene.util.RamUsageEstimator;
 import org.apache.lucene.util.SetOnce;
 import org.elasticsearch.ExceptionsHelper;
@@ -283,7 +284,7 @@ protected static ShardFieldStats shardFieldStats(List<LeafReaderContext> leaves)
         int totalFields = 0;
         long usages = 0;
         long totalPostingBytes = 0;
-        long liveDocsBytes = 0;
+        long totalLiveDocsBytes = 0;
         for (LeafReaderContext leaf : leaves) {
             numSegments++;
             var fieldInfos = leaf.reader().getFieldInfos();
@@ -312,18 +313,23 @@ protected static ShardFieldStats shardFieldStats(List<LeafReaderContext> leaves)
                         var liveDocs = segmentReader.getLiveDocs();
                         if (liveDocs != null) {
                             assert validateLiveDocsClass(liveDocs);
-                            // Would prefer to use FixedBitSet#ramBytesUsed() however FixedBits / Bits interface don't expose that.
-                            // This almost does what FixedBitSet#ramBytesUsed() does, liveDocs.length() returns the length of the bits long
-                            // array
-                            liveDocsBytes += RamUsageEstimator.alignObjectSize(
-                                (long) RamUsageEstimator.NUM_BYTES_ARRAY_HEADER + (liveDocs.length() / 8L)
-                            );
+                            long liveDocsBytes = getLiveDocsBytes(liveDocs);
+                            totalLiveDocsBytes += liveDocsBytes;
                         }
                     }
                 }
             }
         }
-        return new ShardFieldStats(numSegments, totalFields, usages, totalPostingBytes, liveDocsBytes);
+        return new ShardFieldStats(numSegments, totalFields, usages, totalPostingBytes, totalLiveDocsBytes);
+    }
+
+    // Would prefer to use FixedBitSet#ramBytesUsed() however FixedBits / Bits interface don't expose that.
+    // This simulates FixedBitSet#ramBytesUsed() does:
+    private static long getLiveDocsBytes(Bits liveDocs) {
+        int words = FixedBitSet.bits2words(liveDocs.length());
+        return ShardFieldStats.FIXED_BITSET_BASE_RAM_BYTES_USED + RamUsageEstimator.alignObjectSize(
+            RamUsageEstimator.sizeOf(RamUsageEstimator.NUM_BYTES_ARRAY_HEADER + (long) Long.BYTES * words)
+        );
     }
 
     private static boolean validateLiveDocsClass(Bits liveDocs) {

server/src/main/java/org/elasticsearch/index/shard/ShardFieldStats.java

@@ -9,6 +9,8 @@
 
 package org.elasticsearch.index.shard;
 
+import org.apache.lucene.util.FixedBitSet;
+import org.apache.lucene.util.RamUsageEstimator;
 import org.elasticsearch.common.util.FeatureFlag;
 
 /**
@@ -25,5 +27,6 @@
 public record ShardFieldStats(int numSegments, int totalFields, long fieldUsages, long postingsInMemoryBytes, long liveDocsBytes) {
 
     public static final FeatureFlag TRACK_LIVE_DOCS_IN_MEMORY_BYTES = new FeatureFlag("track_live_docs_in_memory_bytes");
+    public static final long FIXED_BITSET_BASE_RAM_BYTES_USED = RamUsageEstimator.shallowSizeOfInstance(FixedBitSet.class);
 
 }

server/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java

@@ -27,6 +27,7 @@
 import org.apache.lucene.store.IOContext;
 import org.apache.lucene.util.BytesRef;
 import org.apache.lucene.util.Constants;
+import org.apache.lucene.util.FixedBitSet;
 import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.ExceptionsHelper;
 import org.elasticsearch.action.ActionListener;
@@ -77,6 +78,7 @@
 import org.elasticsearch.index.IndexModule;
 import org.elasticsearch.index.IndexSettings;
 import org.elasticsearch.index.IndexVersion;
+import org.elasticsearch.index.MergePolicyConfig;
 import org.elasticsearch.index.codec.CodecService;
 import org.elasticsearch.index.codec.TrackingPostingsInMemoryBytesCodec;
 import org.elasticsearch.index.engine.CommitStats;
@@ -1981,7 +1983,10 @@ public void testShardFieldStats() throws IOException {
     }
 
     public void testShardFieldStatsWithDeletes() throws IOException {
-        Settings settings = Settings.builder().put(IndexSettings.INDEX_REFRESH_INTERVAL_SETTING.getKey(), TimeValue.MINUS_ONE).build();
+        Settings settings = Settings.builder()
+            .put(MergePolicyConfig.INDEX_MERGE_ENABLED, false)
+            .put(IndexSettings.INDEX_REFRESH_INTERVAL_SETTING.getKey(), TimeValue.MINUS_ONE)
+            .build();
         IndexShard shard = newShard(true, settings);
         assertNull(shard.getShardFieldStats());
         recoverShardFromStore(shard);
@@ -2010,8 +2015,14 @@ public void testShardFieldStatsWithDeletes() throws IOException {
         stats = shard.getShardFieldStats();
         // More segments because delete operation is stored in the new segment for replication purposes.
         assertThat(stats.numSegments(), equalTo(2));
-        // Delete op is stored in new segment, but marked as deleted. All segements have live docs:
-        assertThat(stats.liveDocsBytes(), equalTo(liveDocsTrackingEnabled ? 40L : 0L));
+        long expectedLiveDocsSize = 0;
+        if (liveDocsTrackingEnabled) {
+            // Delete op is stored in new segment, but marked as deleted. All segements have live docs:
+            expectedLiveDocsSize += new FixedBitSet(numDocs).ramBytesUsed();
+            // Second segment the delete operation that is marked as deleted:
+            expectedLiveDocsSize += new FixedBitSet(1).ramBytesUsed();
+        }
+        assertThat(stats.liveDocsBytes(), equalTo(expectedLiveDocsSize));
 
         // delete another doc:
         deleteDoc(shard, "first_1");
@@ -2022,8 +2033,16 @@ public void testShardFieldStatsWithDeletes() throws IOException {
         stats = shard.getShardFieldStats();
         // More segments because delete operation is stored in the new segment for replication purposes.
         assertThat(stats.numSegments(), equalTo(3));
-        // Delete op is stored in new segment, but marked as deleted. All segements have live docs:
-        assertThat(stats.liveDocsBytes(), equalTo(liveDocsTrackingEnabled ? 56L : 0L));
+        expectedLiveDocsSize = 0;
+        if (liveDocsTrackingEnabled) {
+            // Delete op is stored in new segment, but marked as deleted. All segements have live docs:
+            // First segment with deletes
+            expectedLiveDocsSize += new FixedBitSet(numDocs).ramBytesUsed();
+            // Second and third segments the delete operation that is marked as deleted:
+            expectedLiveDocsSize += new FixedBitSet(1).ramBytesUsed();
+            expectedLiveDocsSize += new FixedBitSet(1).ramBytesUsed();
+        }
+        assertThat(stats.liveDocsBytes(), equalTo(expectedLiveDocsSize));
 
         closeShards(shard);
     }