use binary dv and fix tests

Author: martijnvg

server/src/main/java/org/elasticsearch/index/mapper/IgnoredSourceFieldMapper.java

@@ -11,6 +11,7 @@
 
 import org.apache.lucene.document.SortedSetDocValuesField;
 import org.apache.lucene.document.StoredField;
+import org.apache.lucene.index.BinaryDocValues;
 import org.apache.lucene.index.LeafReader;
 import org.apache.lucene.index.SortedSetDocValues;
 import org.apache.lucene.util.BytesRef;
@@ -28,6 +29,7 @@
 import org.elasticsearch.index.IndexSettings;
 import org.elasticsearch.index.IndexVersion;
 import org.elasticsearch.index.IndexVersions;
+import org.elasticsearch.index.fielddata.MultiValuedSortedBinaryDocValues;
 import org.elasticsearch.index.query.SearchExecutionContext;
 import org.elasticsearch.search.lookup.SourceFilter;
 import org.elasticsearch.xcontent.XContentBuilder;
@@ -553,7 +555,7 @@ public BytesRef filterValue(BytesRef value, Function<Map<String, Object>, Map<St
         DOC_VALUES_IGNORED_SOURCE {
             @Override
             public IgnoredSourceLeafLoader createLeafLoader(LeafReader leafReader) throws IOException {
-                SortedSetDocValues docValues = leafReader.getSortedSetDocValues(NAME);
+                var docValues = MultiValuedSortedBinaryDocValues.from(leafReader, NAME);
                 return new IgnoredSourceLeafLoader() {
                     @Override
                     public Map<String, List<NameValue>> loadAllIgnoredFields(
@@ -567,8 +569,7 @@ public Map<String, List<NameValue>> loadAllIgnoredFields(
                         Map<String, List<NameValue>> objectsWithIgnoredFields = new HashMap<>();
                         int count = docValues.docValueCount();
                         for (int i = 0; i < count; i++) {
-                            long ord = docValues.nextOrd();
-                            BytesRef encoded = docValues.lookupOrd(ord);
+                            BytesRef encoded = docValues.nextValue();
                             List<NameValue> nameValues = CoalescedIgnoredSourceEncoding.decode(encoded);
                             assert nameValues.isEmpty() == false;
                             for (var nameValue : nameValues) {
@@ -595,8 +596,7 @@ public Map<String, List<NameValue>> loadSingleIgnoredField(
                         }
                         int count = docValues.docValueCount();
                         for (int i = 0; i < count; i++) {
-                            long ord = docValues.nextOrd();
-                            BytesRef encoded = docValues.lookupOrd(ord);
+                            BytesRef encoded = docValues.nextValue();
                             List<NameValue> nameValues = CoalescedIgnoredSourceEncoding.decode(encoded);
                             assert nameValues.isEmpty() == false;
                             String fieldPath = nameValues.getFirst().name();
@@ -622,8 +622,11 @@ public void writeIgnoredFields(Collection<NameValue> ignoredFieldValues) {
 
                 for (var docEntry : entriesMap.entrySet()) {
                     for (var fieldEntry : docEntry.getValue().entrySet()) {
-                        docEntry.getKey()
-                            .add(new SortedSetDocValuesField(NAME, CoalescedIgnoredSourceEncoding.encode(fieldEntry.getValue())));
+                        MultiValuedBinaryDocValuesField.SeparateCount.addToSeparateCountMultiBinaryFieldInDoc(
+                            docEntry.getKey(),
+                            NAME,
+                            CoalescedIgnoredSourceEncoding.encode(fieldEntry.getValue())
+                        );
                     }
                 }
             }

server/src/test/java/org/elasticsearch/index/mapper/DocCountFieldMapperTests.java

@@ -99,7 +99,6 @@ public void testSyntheticSourceMany() throws IOException {
             }
         }, reader -> {
             SourceLoader loader = mapper.mappingLookup().newSourceLoader(null, SourceFieldMetrics.NOOP);
-            assertThat(loader.requiredStoredFields(), Matchers.contains("_ignored_source"));
             for (LeafReaderContext leaf : reader.leaves()) {
                 int[] docIds = IntStream.range(0, leaf.reader().maxDoc()).toArray();
                 SourceLoader.Leaf sourceLoaderLeaf = loader.leaf(leaf.reader(), docIds);

test/framework/src/main/java/org/elasticsearch/index/mapper/NativeArrayIntegrationTestCase.java

@@ -15,6 +15,9 @@
 import org.apache.lucene.index.FieldInfos;
 import org.apache.lucene.index.IndexableField;
 import org.apache.lucene.index.LeafReader;
+import org.apache.lucene.index.MultiDocValues;
+import org.apache.lucene.index.MultiFields;
+import org.apache.lucene.index.MultiReader;
 import org.apache.lucene.util.BytesRef;
 import org.elasticsearch.action.admin.indices.forcemerge.ForceMergeRequest;
 import org.elasticsearch.action.admin.indices.refresh.RefreshAction;
@@ -236,9 +239,10 @@ public void testSynthesizeRandomArrayInNestedContext() throws Exception {
 
         try (var searcher = indexService.getShard(0).acquireSearcher(getTestName())) {
             var reader = searcher.getDirectoryReader();
-            var document = reader.storedFields().document(0);
-            Set<String> storedFieldNames = new LinkedHashSet<>(document.getFields().stream().map(IndexableField::name).toList());
-            assertThat(storedFieldNames, contains(IgnoredSourceFieldMapper.NAME));
+            var binaryDV = MultiDocValues.getBinaryValues(reader, IgnoredSourceFieldMapper.NAME);
+            assertThat(binaryDV, notNullValue());
+            binaryDV.advanceExact(0);
+            assertThat(binaryDV.binaryValue(), notNullValue());
             assertThat(FieldInfos.getMergedFieldInfos(reader).fieldInfo("parent.field.offsets"), nullValue());
         }
     }
@@ -376,7 +380,7 @@ protected void verifySyntheticObjectArray(List<List<Object[]>> documents) throws
                 var document = reader.storedFields().document(i);
                 // Verify that there is ignored source because of leaf array being wrapped by object array:
                 List<String> storedFieldNames = document.getFields().stream().map(IndexableField::name).toList();
-                assertThat(storedFieldNames, contains("_id", IgnoredSourceFieldMapper.NAME));
+                assertThat(storedFieldNames, contains("_id"));
 
                 // Verify that there is no offset field:
                 LeafReader leafReader = reader.leaves().get(0).reader();