Include doc value skippers in disk usage analyzer

Author: martijnvg

server/src/main/java/org/elasticsearch/action/admin/indices/diskusage/IndexDiskUsageAnalyzer.java

@@ -289,6 +289,16 @@ void analyzeDocValues(SegmentReader reader, IndexDiskUsageStats stats) throws IO
                     throw new IllegalStateException("Unknown docValues type [" + dvType + "]");
                 }
             }
+            switch (field.docValuesSkipIndexType()) {
+                case NONE -> {}
+                case RANGE -> {
+                    var skipper = docValuesReader.getSkipper(field);
+                    while(skipper.maxDocID(0) != DocIdSetIterator.NO_MORE_DOCS) {
+                        skipper.advance(skipper.maxDocID(0) + 1);
+                    }
+                }
+                default -> throw new IllegalStateException("Unknown skipper type [" + field.docValuesSkipIndexType() + "]");
+            }
             stats.addDocValues(field.name, directory.getBytesRead());
         }
     }

server/src/test/java/org/elasticsearch/action/admin/indices/diskusage/IndexDiskUsageAnalyzerTests.java

@@ -9,6 +9,7 @@
 
 package org.elasticsearch.action.admin.indices.diskusage;
 
+import org.apache.lucene.codecs.Codec;
 import org.apache.lucene.codecs.DocValuesFormat;
 import org.apache.lucene.codecs.KnnVectorsFormat;
 import org.apache.lucene.codecs.PostingsFormat;
@@ -67,7 +68,10 @@
 import org.apache.lucene.util.FixedBitSet;
 import org.elasticsearch.common.lucene.Lucene;
 import org.elasticsearch.core.IOUtils;
+import org.elasticsearch.index.codec.Elasticsearch900Lucene101Codec;
 import org.elasticsearch.index.codec.postings.ES812PostingsFormat;
+import org.elasticsearch.index.codec.tsdb.ES87TSDBDocValuesFormat;
+import org.elasticsearch.index.codec.zstd.Zstd814StoredFieldsFormat;
 import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.index.store.LuceneFilesExtensions;
 import org.elasticsearch.test.ESTestCase;
@@ -405,6 +409,21 @@ public void testMixedFields() throws Exception {
         }
     }
 
+    public void testDocValuesFieldWithDocValueSkippers() throws Exception {
+        try (Directory dir = createNewDirectory()) {
+            var codecMode = randomFrom(CodecMode.values());
+            indexRandomly(dir, codecMode, between(100, 1000), doc -> addRandomDocValuesField(doc, true));
+            final IndexDiskUsageStats stats = IndexDiskUsageAnalyzer.analyze(testShardId(), lastCommit(dir), () -> {});
+            logger.info("--> stats {}", stats);
+            try (Directory perFieldDir = createNewDirectory()) {
+                rewriteIndexWithPerFieldCodec(dir, codecMode, perFieldDir);
+                final IndexDiskUsageStats perFieldStats = collectPerFieldStats(perFieldDir);
+                assertStats(stats, perFieldStats);
+                assertStats(IndexDiskUsageAnalyzer.analyze(testShardId(), lastCommit(perFieldDir), () -> {}), perFieldStats);
+            }
+        }
+    }
+
     private static void addFieldsToDoc(Document doc, IndexableField[] fields) {
         for (IndexableField field : fields) {
             doc.add(field);
@@ -442,23 +461,27 @@ static void indexRandomly(Directory directory, CodecMode codecMode, int numDocs,
         }
     }
 
-    static void addRandomDocValuesField(Document doc) {
+    static void addRandomDocValuesField(Document doc, boolean indexed) {
         if (randomBoolean()) {
-            doc.add(new NumericDocValuesField("ndv", random().nextInt(1024)));
+            int val = random().nextInt(1024);
+            doc.add(indexed ? NumericDocValuesField.indexedField("ndv", val) : new NumericDocValuesField("ndv", val));
         }
-        if (randomBoolean()) {
+        if (randomBoolean() && indexed == false) {
             doc.add(new BinaryDocValuesField("bdv", new BytesRef(randomAlphaOfLength(3))));
         }
         if (randomBoolean()) {
-            doc.add(new SortedDocValuesField("sdv", new BytesRef(randomAlphaOfLength(3))));
+            var value = new BytesRef(randomAlphaOfLength(3));
+            doc.add(indexed ? SortedDocValuesField.indexedField("sdv", value) : new SortedDocValuesField("sdv", value));
         }
         int numValues = random().nextInt(5);
         for (int i = 0; i < numValues; ++i) {
-            doc.add(new SortedSetDocValuesField("ssdv", new BytesRef(randomAlphaOfLength(3))));
+            var value = new BytesRef(randomAlphaOfLength(3));
+            doc.add(indexed ? SortedSetDocValuesField.indexedField("ssdv", value) : new SortedSetDocValuesField("ssdv", value));
         }
         numValues = random().nextInt(5);
         for (int i = 0; i < numValues; ++i) {
-            doc.add(new SortedNumericDocValuesField("sndv", random().nextInt(1024)));
+            int value = random().nextInt(1024);
+            doc.add(indexed ? SortedNumericDocValuesField.indexedField("sndv", value) : new SortedNumericDocValuesField("sndv", value));
         }
     }
 
@@ -535,7 +558,7 @@ private static float[] randomVector(int dimension) {
 
     static void addRandomFields(Document doc) {
         if (randomBoolean()) {
-            addRandomDocValuesField(doc);
+            addRandomDocValuesField(doc, false);
         }
         if (randomBoolean()) {
             addRandomPostings(doc);