Change

Author: Tim-Brooks

server/src/main/java/org/elasticsearch/common/io/stream/RecyclerBytesStreamOutput.java

@@ -141,7 +141,7 @@ public void writeVInt(int i) throws IOException {
         }
     }
 
-    protected static int vIntLength(int value) {
+    public static int vIntLength(int value) {
         int leadingZeros = Integer.numberOfLeadingZeros(value);
         if (leadingZeros >= 25) {
             return 1;

server/src/main/java/org/elasticsearch/index/engine/TranslogDirectoryReader.java

@@ -182,8 +182,9 @@ private static LeafReader createInMemoryReader(
         boolean rootDocOnly,
         Translog.Index operation
     ) {
+        final String id = Uid.decodeId(operation.uid());
         final ParsedDocument parsedDocs = documentParser.parseDocument(
-            new SourceToParse(operation.id(), operation.source(), XContentHelper.xContentType(operation.source()), operation.routing()),
+            new SourceToParse(id, operation.source(), XContentHelper.xContentType(operation.source()), operation.routing()),
             mappingLookup
         );
 
@@ -244,7 +245,7 @@ protected StoredFieldsReader doGetSequentialStoredFieldsReader(StoredFieldsReade
                 }
             };
         } catch (IOException e) {
-            throw new EngineException(shardId, "failed to create an in-memory segment for get [" + operation.id() + "]", e);
+            throw new EngineException(shardId, "failed to create an in-memory segment for get [" + id + "]", e);
         }
     }
 
@@ -339,7 +340,7 @@ private static class TranslogLeafReader extends LeafReader {
             this.engineConfig = engineConfig;
             this.onSegmentCreated = onSegmentCreated;
             this.directory = directory;
-            this.uid = Uid.encodeId(operation.id());
+            this.uid = operation.uid();
         }
 
         private LeafReader getDelegate() {

server/src/main/java/org/elasticsearch/index/mapper/Uid.java

@@ -173,6 +173,12 @@ private static String decodeBase64Id(byte[] idBytes, int offset, int length) {
         return Strings.BASE_64_NO_PADDING_URL_ENCODER.encodeToString(idBytes);
     }
 
+    /** Decode an indexed id back to its original form.
+     *  @see #encodeId */
+    public static String decodeId(BytesRef idBytes) {
+        return decodeId(idBytes.bytes, idBytes.offset, idBytes.length);
+    }
+
     /** Decode an indexed id back to its original form.
      *  @see #encodeId */
     public static String decodeId(byte[] idBytes) {

server/src/main/java/org/elasticsearch/index/shard/IndexShard.java

@@ -2130,7 +2130,12 @@ private Engine.Result applyTranslogOperation(Engine engine, Translog.Operation o
                     index.getAutoGeneratedIdTimestamp(),
                     true,
                     origin,
-                    new SourceToParse(index.id(), index.source(), XContentHelper.xContentType(index.source()), index.routing())
+                    new SourceToParse(
+                        Uid.decodeId(index.uid()),
+                        index.source(),
+                        XContentHelper.xContentType(index.source()),
+                        index.routing()
+                    )
                 );
             }
             case DELETE -> {
@@ -2140,7 +2145,7 @@ private Engine.Result applyTranslogOperation(Engine engine, Translog.Operation o
                     delete.seqNo(),
                     delete.primaryTerm(),
                     delete.version(),
-                    delete.id(),
+                    Uid.decodeId(delete.uid()),
                     versionType,
                     UNASSIGNED_SEQ_NO,
                     0,

server/src/main/java/org/elasticsearch/index/translog/Translog.java

@@ -1243,7 +1243,7 @@ public static final class Index extends Operation {
         public static final int FORMAT_REORDERED = FORMAT_NO_DOC_TYPE + 1;
         public static final int SERIALIZATION_FORMAT = FORMAT_REORDERED;
 
-        private final String id;
+        private final BytesRef uid;
         private final long autoGeneratedIdTimestamp;
         private final long version;
         private final BytesReference source;
@@ -1252,15 +1252,15 @@ public static final class Index extends Operation {
         private static Index readFrom(StreamInput in) throws IOException {
             final int format = in.readVInt(); // SERIALIZATION_FORMAT
             assert format >= FORMAT_NO_PARENT : "format was: " + format;
-            String id;
+            BytesRef uid;
             BytesReference source;
             String routing;
             long version;
             long autoGeneratedIdTimestamp;
             long seqNo;
             long primaryTerm;
             if (format < FORMAT_REORDERED) {
-                id = in.readString();
+                uid = Uid.encodeId(in.readString());
                 if (format < FORMAT_NO_DOC_TYPE) {
                     in.readString();
                     // can't assert that this is _doc because pre-8.0 indexes can have any name for a type
@@ -1279,16 +1279,16 @@ private static Index readFrom(StreamInput in) throws IOException {
                 autoGeneratedIdTimestamp = in.readLong();
                 seqNo = in.readLong();
                 primaryTerm = in.readLong();
+                uid = in.readBytesRef();
                 routing = in.readOptionalString();
-                id = in.readString();
                 source = in.readBytesReference();
             }
-            return new Index(id, seqNo, primaryTerm, version, source, routing, autoGeneratedIdTimestamp);
+            return new Index(uid, seqNo, primaryTerm, version, source, routing, autoGeneratedIdTimestamp);
         }
 
         public Index(Engine.Index index, Engine.IndexResult indexResult) {
             this(
-                index.id(),
+                index.uid(),
                 indexResult.getSeqNo(),
                 index.primaryTerm(),
                 indexResult.getVersion(),
@@ -1306,9 +1306,21 @@ public Index(
             BytesReference source,
             String routing,
             long autoGeneratedIdTimestamp
+        ) {
+            this(Uid.encodeId(id), seqNo, primaryTerm, version, source, routing, autoGeneratedIdTimestamp);
+        }
+
+        public Index(
+            BytesRef uid,
+            long seqNo,
+            long primaryTerm,
+            long version,
+            BytesReference source,
+            String routing,
+            long autoGeneratedIdTimestamp
         ) {
             super(seqNo, primaryTerm);
-            this.id = id;
+            this.uid = uid;
             this.source = source;
             this.version = version;
             this.routing = routing;
@@ -1322,14 +1334,14 @@ public Type opType() {
 
         @Override
         public long estimateSize() {
-            return (2 * id.length()) + source.length() + (routing != null ? 2 * routing.length() : 0) + (4 * Long.BYTES); // timestamp,
+            return uid.length + source.length() + (routing != null ? 2 * routing.length() : 0) + (4 * Long.BYTES); // timestamp,
             // seq_no,
             // primary_term,
             // and version
         }
 
-        public String id() {
-            return this.id;
+        public BytesRef uid() {
+            return uid;
         }
 
         public String routing() {
@@ -1351,8 +1363,8 @@ protected void writeHeader(int format, StreamOutput out) throws IOException {
             out.writeLong(autoGeneratedIdTimestamp);
             out.writeLong(seqNo);
             out.writeLong(primaryTerm);
+            out.writeBytesRef(uid);
             out.writeOptionalString(routing);
-            out.writeString(id);
             out.writeVInt(source == null ? 0 : source.length());
         }
 
@@ -1363,7 +1375,7 @@ public void writeBody(final StreamOutput out) throws IOException {
                 : FORMAT_NO_VERSION_TYPE;
             if (format < FORMAT_REORDERED) {
                 out.writeVInt(format);
-                out.writeString(id);
+                out.writeString(Uid.decodeId(uid.bytes, uid.offset, uid.length));
                 if (format < FORMAT_NO_DOC_TYPE) {
                     out.writeString(MapperService.SINGLE_MAPPING_NAME);
                 }
@@ -1396,7 +1408,7 @@ public boolean equals(Object o) {
 
         @Override
         public int hashCode() {
-            int result = id.hashCode();
+            int result = uid.hashCode();
             result = 31 * result + Long.hashCode(seqNo);
             result = 31 * result + Long.hashCode(primaryTerm);
             result = 31 * result + Long.hashCode(version);
@@ -1410,7 +1422,7 @@ public int hashCode() {
         public String toString() {
             return "Index{"
                 + "id='"
-                + id
+                + Uid.decodeId(uid.bytes, uid.offset, uid.length)
                 + '\''
                 + ", seqNo="
                 + seqNo
@@ -1431,7 +1443,7 @@ public static boolean equalsWithoutAutoGeneratedTimestamp(Translog.Index o1, Tra
             if (o1.version != o2.version
                 || o1.seqNo != o2.seqNo
                 || o1.primaryTerm != o2.primaryTerm
-                || o1.id.equals(o2.id) == false
+                || o1.uid.equals(o2.uid) == false
                 || Objects.equals(o1.routing, o2.routing) == false) {
                 return false;
             }
@@ -1475,13 +1487,13 @@ public static final class Delete extends Operation {
         public static final int FORMAT_REORDERED = FORMAT_NO_DOC_TYPE + 1;
         public static final int SERIALIZATION_FORMAT = FORMAT_REORDERED;
 
-        private final String id;
+        private final BytesRef uid;
         private final long version;
 
         private static Delete readFrom(StreamInput in) throws IOException {
             final int format = in.readVInt();// SERIALIZATION_FORMAT
             assert format >= FORMAT_6_0 : "format was: " + format;
-            final String id;
+            final BytesRef uid;
             final long version;
             final long seqNo;
             final long primaryTerm;
@@ -1490,7 +1502,7 @@ private static Delete readFrom(StreamInput in) throws IOException {
                     in.readString();
                     // Can't assert that this is _doc because pre-8.0 indexes can have any name for a type
                 }
-                id = in.readString();
+                uid = Uid.encodeId(in.readString());
                 if (format < FORMAT_NO_DOC_TYPE) {
                     final String docType = in.readString();
                     assert docType.equals(IdFieldMapper.NAME) : docType + " != " + IdFieldMapper.NAME;
@@ -1506,23 +1518,28 @@ private static Delete readFrom(StreamInput in) throws IOException {
                 version = in.readLong();
                 seqNo = in.readLong();
                 primaryTerm = in.readLong();
-                id = in.readString();
+                uid = in.readBytesRef();
             }
-            return new Delete(id, seqNo, primaryTerm, version);
+            return new Delete(uid, seqNo, primaryTerm, version);
         }
 
         public Delete(Engine.Delete delete, Engine.DeleteResult deleteResult) {
-            this(delete.id(), deleteResult.getSeqNo(), delete.primaryTerm(), deleteResult.getVersion());
+            this(delete.uid(), deleteResult.getSeqNo(), delete.primaryTerm(), deleteResult.getVersion());
         }
 
         /** utility for testing */
         public Delete(String id, long seqNo, long primaryTerm) {
             this(id, seqNo, primaryTerm, Versions.MATCH_ANY);
         }
 
+        /** utility for testing */
         public Delete(String id, long seqNo, long primaryTerm, long version) {
+            this(Uid.encodeId(id), seqNo, primaryTerm, version);
+        }
+
+        public Delete(BytesRef uid, long seqNo, long primaryTerm, long version) {
             super(seqNo, primaryTerm);
-            this.id = Objects.requireNonNull(id);
+            this.uid = Objects.requireNonNull(uid);
             this.version = version;
         }
 
@@ -1533,7 +1550,7 @@ public Type opType() {
 
         @Override
         public long estimateSize() {
-            return (2 * id.length()) + (3 * Long.BYTES); // seq_no, primary_term, and version;
+            return uid.length + (3 * Long.BYTES); // seq_no, primary_term, and version;
         }
 
         @Override
@@ -1542,11 +1559,11 @@ protected void writeHeader(int format, StreamOutput out) throws IOException {
             out.writeLong(version);
             out.writeLong(seqNo);
             out.writeLong(primaryTerm);
-            out.writeString(id);
+            out.writeBytesRef(uid);
         }
 
-        public String id() {
-            return id;
+        public BytesRef uid() {
+            return uid;
         }
 
         public long version() {
@@ -1563,10 +1580,10 @@ public void writeBody(final StreamOutput out) throws IOException {
                 if (format < FORMAT_NO_DOC_TYPE) {
                     out.writeString(MapperService.SINGLE_MAPPING_NAME);
                 }
-                out.writeString(id);
+                out.writeString(Uid.decodeId(uid));
                 if (format < FORMAT_NO_DOC_TYPE) {
                     out.writeString(IdFieldMapper.NAME);
-                    out.writeBytesRef(Uid.encodeId(id));
+                    out.writeBytesRef(uid);
                 }
                 out.writeLong(version);
                 out.writeLong(seqNo);
@@ -1587,12 +1604,12 @@ public boolean equals(Object o) {
 
             Delete delete = (Delete) o;
 
-            return id.equals(delete.id) && seqNo == delete.seqNo && primaryTerm == delete.primaryTerm && version == delete.version;
+            return uid.equals(delete.uid) && seqNo == delete.seqNo && primaryTerm == delete.primaryTerm && version == delete.version;
         }
 
         @Override
         public int hashCode() {
-            int result = id.hashCode();
+            int result = uid.hashCode();
             result += 31 * Long.hashCode(seqNo);
             result = 31 * result + Long.hashCode(primaryTerm);
             result = 31 * result + Long.hashCode(version);
@@ -1601,7 +1618,16 @@ public int hashCode() {
 
         @Override
         public String toString() {
-            return "Delete{" + "id='" + id + "', seqNo=" + seqNo + ", primaryTerm=" + primaryTerm + ", version=" + version + '}';
+            return "Delete{"
+                + "id='"
+                + Uid.decodeId(uid)
+                + "', seqNo="
+                + seqNo
+                + ", primaryTerm="
+                + primaryTerm
+                + ", version="
+                + version
+                + '}';
         }
     }
 

server/src/main/java/org/elasticsearch/index/translog/TranslogHeaderWriter.java

@@ -12,6 +12,7 @@
 import org.apache.lucene.util.BytesRef;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.io.stream.RecyclerBytesStreamOutput;
+import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.util.ByteUtils;
 
 import java.io.IOException;
@@ -35,15 +36,24 @@ private TranslogHeaderWriter() {
     }
 
     public static void writeIndexHeader(RecyclerBytesStreamOutput buffer, Translog.Index index) throws IOException {
-        BytesRef page = buffer.tryGetPageForWrite(FIXED_INDEX_HEADER_SIZE);
+        int uidLen = index.uid().length;
+        int uidVIntLen = RecyclerBytesStreamOutput.vIntLength(uidLen);
+        BytesRef page = buffer.tryGetPageForWrite(FIXED_INDEX_HEADER_SIZE + uidLen + uidVIntLen);
         if (page != null) {
-            writeFastIndexHeader(buffer, index, page);
+            writeFastIndexHeader(buffer, index, page, uidLen, uidVIntLen);
         } else {
             writeSlowIndexHeader(buffer, index);
         }
     }
 
-    private static void writeFastIndexHeader(RecyclerBytesStreamOutput buffer, Translog.Index index, BytesRef page) throws IOException {
+    private static void writeFastIndexHeader(
+        RecyclerBytesStreamOutput buffer,
+        Translog.Index index,
+        BytesRef page,
+        int uidLen,
+        int uidVIntLen
+    ) throws IOException {
+        BytesRef uid = index.uid();
         String routing = index.routing();
 
         int off = page.offset;
@@ -55,14 +65,15 @@ private static void writeFastIndexHeader(RecyclerBytesStreamOutput buffer, Trans
         ByteUtils.writeLongBE(index.getAutoGeneratedIdTimestamp(), bytes, off + 14);
         ByteUtils.writeLongBE(index.seqNo(), bytes, off + 22);
         ByteUtils.writeLongBE(index.primaryTerm(), bytes, off + 30);
-        bytes[off + 38] = index.routing() == null ? (byte) 0 : (byte) 1;
+        StreamOutput.putVInt(bytes, uid.length, off + 38);
+        System.arraycopy(uid.bytes, uid.offset, bytes, off + 38 + uidVIntLen, uid.length);
+        bytes[off + 38 + uidVIntLen + uidLen] = index.routing() == null ? (byte) 0 : (byte) 1;
 
         long variableLengthStart = buffer.position();
         // Write variable length items in header
         if (routing != null) {
             buffer.writeString(routing);
         }
-        buffer.writeString(index.id());
 
         BytesReference source = index.source();
         int sourceLength = source == null ? 0 : source.length();
@@ -109,7 +120,7 @@ private static void writeFastDeleteHeader(RecyclerBytesStreamOutput buffer, Tran
 
         long variableLengthStart = buffer.position();
         // Write variable length items in header
-        buffer.writeString(delete.id());
+        buffer.writeBytesRef(delete.uid());
         int variableLengthSize = (int) (buffer.position() - variableLengthStart);
         // The total operation size is the header size + 4 bytes for checksum
         int sizeOfOperation = FIXED_DELETE_HEADER_SIZE - Integer.BYTES + variableLengthSize + Integer.BYTES;