Investigate using doc values for logsdb _id

server/src/main/java/org/elasticsearch/common/lucene/uid/PerThreadIDVersionAndSeqNoLookup.java

@@ -51,6 +51,7 @@ final class PerThreadIDVersionAndSeqNoLookup {
     // we keep it around for now, to reduce the amount of e.g. hash lookups by field and stuff
 
     private final TermsEnum termsEnum;
+    private final LeafReader reader;
 
     /** Reused for iteration (when the term exists) */
     private PostingsEnum docsEnum;
@@ -66,22 +67,25 @@ final class PerThreadIDVersionAndSeqNoLookup {
      * Initialize lookup for the provided segment
      */
     PerThreadIDVersionAndSeqNoLookup(LeafReader reader, boolean trackReaderKey, boolean loadTimestampRange) throws IOException {
+        this.reader = reader;
         final Terms terms = reader.terms(IdFieldMapper.NAME);
         if (terms == null) {
-            // If a segment contains only no-ops, it does not have _uid but has both _soft_deletes and _tombstone fields.
-            final NumericDocValues softDeletesDV = reader.getNumericDocValues(Lucene.SOFT_DELETES_FIELD);
-            final NumericDocValues tombstoneDV = reader.getNumericDocValues(SeqNoFieldMapper.TOMBSTONE_NAME);
-            // this is a special case when we pruned away all IDs in a segment since all docs are deleted.
-            final boolean allDocsDeleted = (softDeletesDV != null && reader.numDocs() == 0);
-            if ((softDeletesDV == null || tombstoneDV == null) && allDocsDeleted == false) {
-                throw new IllegalArgumentException(
-                    "reader does not have _uid terms but not a no-op segment; "
-                        + "_soft_deletes ["
-                        + softDeletesDV
-                        + "], _tombstone ["
-                        + tombstoneDV
-                        + "]"
-                );
+            if (reader.getSortedDocValues(IdFieldMapper.NAME) == null) {
+                // If a segment contains only no-ops, it does not have _uid but has both _soft_deletes and _tombstone fields.
+                final NumericDocValues softDeletesDV = reader.getNumericDocValues(Lucene.SOFT_DELETES_FIELD);
+                final NumericDocValues tombstoneDV = reader.getNumericDocValues(SeqNoFieldMapper.TOMBSTONE_NAME);
+                // this is a special case when we pruned away all IDs in a segment since all docs are deleted.
+                final boolean allDocsDeleted = (softDeletesDV != null && reader.numDocs() == 0);
+                if ((softDeletesDV == null || tombstoneDV == null) && allDocsDeleted == false) {
+                    throw new IllegalArgumentException(
+                        "reader does not have _uid terms but not a no-op segment; "
+                            + "_soft_deletes ["
+                            + softDeletesDV
+                            + "], _tombstone ["
+                            + tombstoneDV
+                            + "]"
+                    );
+                }
             }
             termsEnum = null;
         } else {
@@ -171,7 +175,27 @@ private int getDocID(BytesRef id, LeafReaderContext context) throws IOException
             }
             return docID;
         } else {
-            return DocIdSetIterator.NO_MORE_DOCS;
+            var idDocValues = reader.getSortedDocValues(IdFieldMapper.NAME);
+            if (idDocValues == null) {
+                return DocIdSetIterator.NO_MORE_DOCS;
+            }
+
+            final Bits liveDocs = context.reader().getLiveDocs();
+            int docID = DocIdSetIterator.NO_MORE_DOCS;
+            // TODO can we use skippers here?
+            // there may be more than one matching docID, in the case of nested docs, so we want the last one:
+            for (int doc = idDocValues.nextDoc(); doc != DocIdSetIterator.NO_MORE_DOCS; doc = idDocValues.nextDoc()) {
+                if (liveDocs != null && liveDocs.get(doc) == false) {
+                    continue;
+                }
+
+                int ord = idDocValues.ordValue();
+                var possibleMatch = idDocValues.lookupOrd(ord);
+                if (id.bytesEquals(possibleMatch)) {
+                    docID = doc;
+                }
+            }
+            return docID;
         }
     }
 

server/src/main/java/org/elasticsearch/index/IndexMode.java

@@ -26,6 +26,7 @@
 import org.elasticsearch.index.mapper.FieldMapper;
 import org.elasticsearch.index.mapper.IdFieldMapper;
 import org.elasticsearch.index.mapper.KeywordFieldMapper;
+import org.elasticsearch.index.mapper.LogsdbIdFieldMapper;
 import org.elasticsearch.index.mapper.MapperService;
 import org.elasticsearch.index.mapper.MappingLookup;
 import org.elasticsearch.index.mapper.MappingParserContext;
@@ -264,12 +265,12 @@ public CompressedXContent getDefaultMapping(final IndexSettings indexSettings) {
 
         @Override
         public IdFieldMapper buildIdFieldMapper(BooleanSupplier fieldDataEnabled) {
-            return new ProvidedIdFieldMapper(fieldDataEnabled);
+            return new LogsdbIdFieldMapper();
         }
 
         @Override
         public IdFieldMapper idFieldMapperWithoutFieldData() {
-            return ProvidedIdFieldMapper.NO_FIELD_DATA;
+            return new LogsdbIdFieldMapper();
         }
 
         @Override

server/src/main/java/org/elasticsearch/index/engine/InternalEngine.java

@@ -1591,6 +1591,7 @@ private boolean assertDocDoesNotExist(final Index index, final boolean allowDele
     }
 
     private void updateDocs(final BytesRef uid, final List<LuceneDocument> docs, final IndexWriter indexWriter) throws IOException {
+        // TODO
         final Term uidTerm = new Term(IdFieldMapper.NAME, uid);
         if (docs.size() > 1) {
             indexWriter.softUpdateDocuments(uidTerm, docs, softDeletesField);
@@ -1822,7 +1823,9 @@ private DeleteResult deleteInLucene(Delete delete, DeletionStrategy plan) throws
             if (plan.addStaleOpToLucene || plan.currentlyDeleted) {
                 indexWriter.addDocument(doc);
             } else {
+                // TODO
                 indexWriter.softUpdateDocument(new Term(IdFieldMapper.NAME, delete.uid()), doc, softDeletesField);
+
             }
             return new DeleteResult(
                 plan.versionOfDeletion,
@@ -3460,6 +3463,7 @@ private void restoreVersionMapAndCheckpointTracker(DirectoryReader directoryRead
                 continue;
             }
             final CombinedDocValues dv = new CombinedDocValues(leaf.reader());
+            // TODO
             final IdStoredFieldLoader idFieldLoader = new IdStoredFieldLoader(leaf.reader());
             final DocIdSetIterator iterator = scorer.iterator();
             int docId;

server/src/main/java/org/elasticsearch/index/mapper/BlockDocValuesReader.java

@@ -549,6 +549,103 @@ public String toString() {
         }
     }
 
+    public static class IdDocValuesBlockLoader extends DocValuesBlockLoader {
+        private final String fieldName;
+
+        public IdDocValuesBlockLoader(String fieldName) {
+            this.fieldName = fieldName;
+        }
+
+        @Override
+        public BytesRefBuilder builder(BlockFactory factory, int expectedCount) {
+            return factory.bytesRefs(expectedCount);
+        }
+
+        @Override
+        public AllReader reader(LeafReaderContext context) throws IOException {
+            SortedDocValues sorted = DocValues.getSorted(context.reader(), fieldName);
+            return new IdDocValuesReader(sorted);
+        }
+
+        @Override
+        public boolean supportsOrdinals() {
+            return true;
+        }
+
+        @Override
+        public SortedSetDocValues ordinals(LeafReaderContext context) throws IOException {
+            return DocValues.getSortedSet(context.reader(), fieldName);
+        }
+
+        @Override
+        public String toString() {
+            return "BytesRefsFromOrds[" + fieldName + "]";
+        }
+    }
+
+    private static class IdDocValuesReader extends BlockDocValuesReader {
+        private final SortedDocValues values;
+
+        IdDocValuesReader(SortedDocValues values) {
+            this.values = values;
+        }
+
+        private BlockLoader.Block readSingleDoc(BlockFactory factory, int docId) throws IOException {
+            if (values.advanceExact(docId)) {
+                BytesRef v = values.lookupOrd(values.ordValue());
+                return factory.constantBytes(decode(v));
+            } else {
+                return factory.constantNulls();
+            }
+        }
+
+        private static BytesRef decode(BytesRef v) {
+            String s = Uid.decodeId(v.bytes, v.offset, v.length);
+            return new BytesRef(s);
+        }
+
+        @Override
+        public BlockLoader.Block read(BlockFactory factory, Docs docs) throws IOException {
+            if (docs.count() == 1) {
+                return readSingleDoc(factory, docs.get(0));
+            }
+            try (BytesRefBuilder builder = factory.bytesRefs(docs.count())) {
+                for (int i = 0; i < docs.count(); i++) {
+                    int doc = docs.get(i);
+                    if (doc < values.docID()) {
+                        throw new IllegalStateException("docs within same block must be in order");
+                    }
+                    if (values.advanceExact(doc)) {
+                        builder.appendBytesRef(decode(values.lookupOrd(values.ordValue())));
+                    } else {
+                        builder.appendNull();
+                    }
+                }
+                return builder.build();
+            }
+        }
+
+        @Override
+        public void read(int docId, BlockLoader.StoredFields storedFields, Builder builder) throws IOException {
+            if (values.advanceExact(docId)) {
+                BytesRef encoded = values.lookupOrd(values.ordValue());
+                ((BytesRefBuilder) builder).appendBytesRef(decode(encoded));
+            } else {
+                builder.appendNull();
+            }
+        }
+
+        @Override
+        public int docId() {
+            return values.docID();
+        }
+
+        @Override
+        public String toString() {
+            return "BlockDocValuesReader.IdDocValuesReader";
+        }
+    }
+
     private static class SingletonOrdinals extends BlockDocValuesReader {
         private final SortedDocValues ordinals;
 

server/src/main/java/org/elasticsearch/index/mapper/DocumentParserContext.java

@@ -10,6 +10,7 @@
 package org.elasticsearch.index.mapper;
 
 import org.apache.lucene.document.Field;
+import org.apache.lucene.document.SortedDocValuesField;
 import org.apache.lucene.document.StringField;
 import org.apache.lucene.index.IndexableField;
 import org.apache.lucene.util.BytesRef;
@@ -717,9 +718,13 @@ public final DocumentParserContext createNestedContext(NestedObjectMapper nested
         // and inner hits to fail or yield incorrect results.
         IndexableField idField = doc.getParent().getField(IdFieldMapper.NAME);
         if (idField != null) {
-            // We just need to store the id as indexed field, so that IndexWriter#deleteDocuments(term) can then
-            // delete it when the root document is deleted too.
-            doc.add(new StringField(IdFieldMapper.NAME, idField.binaryValue(), Field.Store.NO));
+            if (indexSettings().getMode() == IndexMode.LOGSDB) {
+                doc.add(SortedDocValuesField.indexedField(IdFieldMapper.NAME, idField.binaryValue()));
+            } else {
+                // We just need to store the id as indexed field, so that IndexWriter#deleteDocuments(term) can then
+                // delete it when the root document is deleted too.
+                doc.add(new StringField(IdFieldMapper.NAME, idField.binaryValue(), Field.Store.NO));
+            }
         } else if (indexSettings().getMode() == IndexMode.TIME_SERIES) {
             // For time series indices, the _id is generated from the _tsid, which in turn is generated from the values of the configured
             // routing fields. At this point in document parsing, we can't guarantee that we've parsed all the routing fields yet, so the