Fix the translog operation asserter to check for map equivalence rather than byte per byte if fields are synthetic

Author: jimczi

server/src/main/java/org/elasticsearch/index/engine/TranslogOperationAsserter.java

@@ -41,8 +41,10 @@ public boolean assertSameIndexOperation(Translog.Index o1, Translog.Index o2) th
                 if (engineConfig.getIndexSettings().isRecoverySourceSyntheticEnabled()
                     || engineConfig.getMapperService().mappingLookup().inferenceFields().isEmpty() == false
                     || engineConfig.getMapperService().mappingLookup().syntheticVectorFields().isEmpty() == false) {
-                    return super.assertSameIndexOperation(synthesizeSource(engineConfig, o1), o2)
-                        || super.assertSameIndexOperation(o1, synthesizeSource(engineConfig, o2));
+                    // for synthetic source and synthetic fields, we check that the resulting source map is equivalent
+                    // since ordering might not be preserved.
+                    return Translog.Index.equalsWithoutAutoGeneratedTimestamp(synthesizeSource(engineConfig, o1), o2, false)
+                        || Translog.Index.equalsWithoutAutoGeneratedTimestamp(o1, synthesizeSource(engineConfig, o2), false);
                 }
                 return false;
             }
@@ -99,6 +101,6 @@ static Translog.Snapshot newSnapshot(EngineConfig engineConfig, Translog.Index o
     }
 
     public boolean assertSameIndexOperation(Translog.Index o1, Translog.Index o2) throws IOException {
-        return Translog.Index.equalsWithoutAutoGeneratedTimestamp(o1, o2);
+        return Translog.Index.equalsWithoutAutoGeneratedTimestamp(o1, o2, true);
     }
 }

server/src/main/java/org/elasticsearch/index/translog/Translog.java

@@ -22,6 +22,7 @@
 import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.common.lucene.uid.Versions;
 import org.elasticsearch.common.util.BigArrays;
+import org.elasticsearch.common.xcontent.XContentHelper;
 import org.elasticsearch.core.IOUtils;
 import org.elasticsearch.core.Nullable;
 import org.elasticsearch.core.Releasable;
@@ -35,6 +36,8 @@
 import org.elasticsearch.index.shard.AbstractIndexShardComponent;
 import org.elasticsearch.index.shard.IndexShardComponent;
 import org.elasticsearch.index.shard.ShardId;
+import org.elasticsearch.search.lookup.Source;
+import org.elasticsearch.xcontent.XContentParserConfiguration;
 
 import java.io.Closeable;
 import java.io.EOFException;
@@ -1226,9 +1229,9 @@ public Type opType() {
         @Override
         public long estimateSize() {
             return (2 * id.length()) + source.length() + (routing != null ? 2 * routing.length() : 0) + (4 * Long.BYTES); // timestamp,
-                                                                                                                          // seq_no,
-                                                                                                                          // primary_term,
-                                                                                                                          // and version
+            // seq_no,
+            // primary_term,
+            // and version
         }
 
         public String id() {
@@ -1275,7 +1278,7 @@ public boolean equals(Object o) {
             }
 
             Index other = (Index) o;
-            return autoGeneratedIdTimestamp == other.autoGeneratedIdTimestamp && equalsWithoutAutoGeneratedTimestamp(this, other);
+            return autoGeneratedIdTimestamp == other.autoGeneratedIdTimestamp && equalsWithoutAutoGeneratedTimestamp(this, other, true);
         }
 
         @Override
@@ -1311,15 +1314,43 @@ public long getAutoGeneratedIdTimestamp() {
             return autoGeneratedIdTimestamp;
         }
 
-        public static boolean equalsWithoutAutoGeneratedTimestamp(Translog.Index o1, Translog.Index o2) {
-            return o1.version == o2.version
-                && o1.seqNo == o2.seqNo
-                && o1.primaryTerm == o2.primaryTerm
-                && o1.id.equals(o2.id)
-                && o1.source.equals(o2.source)
-                && Objects.equals(o1.routing, o2.routing);
-        }
+        public static boolean equalsWithoutAutoGeneratedTimestamp(Translog.Index o1, Translog.Index o2, boolean checkSourceBytes) {
+            if (o1.version != o2.version
+                || o1.seqNo != o2.seqNo
+                || o1.primaryTerm != o2.primaryTerm
+                || o1.id.equals(o2.id) == false
+                || Objects.equals(o1.routing, o2.routing) == false) {
+                return false;
+            }
+
+            if (checkSourceBytes) {
+                return o1.source.equals(o2.source);
+            }
 
+            var s1 = Source.fromBytes(o1.source);
+            var s2 = Source.fromBytes(o2.source);
+            try (
+                var actualParser = XContentHelper.createParserNotCompressed(
+                    XContentParserConfiguration.EMPTY,
+                    s1.internalSourceRef(),
+                    s1.sourceContentType()
+                )
+            ) {
+                var actualMap = actualParser.map();
+                try (
+                    var expectedParser = XContentHelper.createParserNotCompressed(
+                        XContentParserConfiguration.EMPTY,
+                        s2.internalSourceRef(),
+                        s2.sourceContentType()
+                    )
+                ) {
+                    var expectedMap = expectedParser.map();
+                    return expectedMap.equals(actualMap);
+                }
+            } catch (IOException exc) {
+                return false;
+            }
+        }
     }
 
     public static final class Delete extends Operation {