diff --git a/server/src/main/java/org/elasticsearch/action/index/IndexRequest.java b/server/src/main/java/org/elasticsearch/action/index/IndexRequest.java index e3fe113c2b976..c9ef82d9c6d40 100644 --- a/server/src/main/java/org/elasticsearch/action/index/IndexRequest.java +++ b/server/src/main/java/org/elasticsearch/action/index/IndexRequest.java @@ -230,7 +230,7 @@ public IndexRequest(@Nullable ShardId shardId, StreamInput in) throws IOExceptio } // else default value is true if (in.getTransportVersion().supports(INDEX_REQUEST_INCLUDE_TSID)) { - tsid = in.readBytesRefOrNullIfEmpty(); + tsid = in.readOptionalBytesRef(); } } diff --git a/server/src/main/java/org/elasticsearch/common/io/stream/StreamInput.java b/server/src/main/java/org/elasticsearch/common/io/stream/StreamInput.java index b45f3ccbdc5f3..607e9121a9919 100644 --- a/server/src/main/java/org/elasticsearch/common/io/stream/StreamInput.java +++ b/server/src/main/java/org/elasticsearch/common/io/stream/StreamInput.java @@ -203,7 +203,7 @@ public BytesRef readBytesRef() throws IOException { return readBytesRef(length); } - public @Nullable BytesRef readBytesRefOrNullIfEmpty() throws IOException { + public @Nullable BytesRef readOptionalBytesRef() throws IOException { int length = readArraySize(); if (length == 0) { return null; diff --git a/server/src/main/java/org/elasticsearch/index/IndexMode.java b/server/src/main/java/org/elasticsearch/index/IndexMode.java index 46c790a4b5b70..08e582353dae1 100644 --- a/server/src/main/java/org/elasticsearch/index/IndexMode.java +++ b/server/src/main/java/org/elasticsearch/index/IndexMode.java @@ -35,6 +35,7 @@ import org.elasticsearch.index.mapper.RoutingFields; import org.elasticsearch.index.mapper.RoutingPathFields; import org.elasticsearch.index.mapper.SourceFieldMapper; +import org.elasticsearch.index.mapper.SourceToParse; import org.elasticsearch.index.mapper.TimeSeriesIdFieldMapper; import org.elasticsearch.index.mapper.TimeSeriesRoutingHashFieldMapper; import org.elasticsearch.index.mapper.TsidExtractingIdFieldMapper; @@ -112,7 +113,7 @@ public IdFieldMapper buildIdFieldMapper(BooleanSupplier fieldDataEnabled) { } @Override - public RoutingFields buildRoutingFields(IndexSettings settings) { + public RoutingFields buildRoutingFields(IndexSettings settings, SourceToParse source) { return RoutingFields.Noop.INSTANCE; } @@ -216,10 +217,19 @@ public IdFieldMapper buildIdFieldMapper(BooleanSupplier fieldDataEnabled) { } @Override - public RoutingFields buildRoutingFields(IndexSettings settings) { + public RoutingFields buildRoutingFields(IndexSettings settings, SourceToParse source) { IndexRouting indexRouting = settings.getIndexRouting(); if (indexRouting instanceof IndexRouting.ExtractFromSource.ForRoutingPath forRoutingPath) { - return new RoutingPathFields(forRoutingPath.builder()); + if (source.tsid() != null) { + // If we already have a tsid, do not extract it from the source again. + // This happens during translog operation replay where we get the tsid value from the translog. + // We don't want to re-create the tsid to avoid that it's different from the original one. + // That could happen, for example, if a new dimension field was added to a non-dynamic mapping + // after the translog operation was created. + return RoutingFields.Noop.INSTANCE; + } else { + return new RoutingPathFields(forRoutingPath.builder()); + } } else if (indexRouting instanceof IndexRouting.ExtractFromSource.ForIndexDimensions) { return RoutingFields.Noop.INSTANCE; } else { @@ -303,7 +313,7 @@ public MetadataFieldMapper timeSeriesRoutingHashFieldMapper() { } @Override - public RoutingFields buildRoutingFields(IndexSettings settings) { + public RoutingFields buildRoutingFields(IndexSettings settings, SourceToParse source) { return RoutingFields.Noop.INSTANCE; } @@ -384,7 +394,7 @@ public IdFieldMapper buildIdFieldMapper(BooleanSupplier fieldDataEnabled) { } @Override - public RoutingFields buildRoutingFields(IndexSettings settings) { + public RoutingFields buildRoutingFields(IndexSettings settings, SourceToParse source) { return RoutingFields.Noop.INSTANCE; } @@ -540,7 +550,7 @@ public String getName() { /** * How {@code time_series_dimension} fields are handled by indices in this mode. */ - public abstract RoutingFields buildRoutingFields(IndexSettings settings); + public abstract RoutingFields buildRoutingFields(IndexSettings settings, SourceToParse source); /** * @return Whether timestamps should be validated for being withing the time range of an index. diff --git a/server/src/main/java/org/elasticsearch/index/engine/Engine.java b/server/src/main/java/org/elasticsearch/index/engine/Engine.java index 5a1c49b54b7ac..edd78e4b73e6f 100644 --- a/server/src/main/java/org/elasticsearch/index/engine/Engine.java +++ b/server/src/main/java/org/elasticsearch/index/engine/Engine.java @@ -1905,6 +1905,11 @@ public String id() { return this.doc.id(); } + @Nullable + public BytesRef tsid() { + return this.doc.tsid(); + } + @Override public TYPE operationType() { return TYPE.INDEX; diff --git a/server/src/main/java/org/elasticsearch/index/engine/LuceneChangesSnapshot.java b/server/src/main/java/org/elasticsearch/index/engine/LuceneChangesSnapshot.java index 54010cab0f3f4..c3ddd869f7147 100644 --- a/server/src/main/java/org/elasticsearch/index/engine/LuceneChangesSnapshot.java +++ b/server/src/main/java/org/elasticsearch/index/engine/LuceneChangesSnapshot.java @@ -273,7 +273,8 @@ private Translog.Operation readDocAsOp(int docIndex) throws IOException { } // TODO: pass the latest timestamp from engine. final long autoGeneratedIdTimestamp = -1; - op = new Translog.Index(id, seqNo, primaryTerm, version, source, fields.routing(), autoGeneratedIdTimestamp); + // TODO get the tsid? + op = new Translog.Index(id, seqNo, primaryTerm, version, source, fields.routing(), autoGeneratedIdTimestamp, null); } } assert fromSeqNo <= op.seqNo() && op.seqNo() <= toSeqNo && lastSeenSeqNo < op.seqNo() diff --git a/server/src/main/java/org/elasticsearch/index/engine/LuceneSyntheticSourceChangesSnapshot.java b/server/src/main/java/org/elasticsearch/index/engine/LuceneSyntheticSourceChangesSnapshot.java index 3b4986c6e17af..406279a9c9354 100644 --- a/server/src/main/java/org/elasticsearch/index/engine/LuceneSyntheticSourceChangesSnapshot.java +++ b/server/src/main/java/org/elasticsearch/index/engine/LuceneSyntheticSourceChangesSnapshot.java @@ -263,7 +263,8 @@ private Translog.Operation createOperation( docRecord.version(), source.internalSourceRef(), fieldLoader.routing(), - -1 // autogenerated timestamp + -1, // autogenerated timestamp + null ); } } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java b/server/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java index 4112f9108d3ee..4f67e049b767e 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java @@ -121,7 +121,8 @@ public ParsedDocument parseDocument(SourceToParse source, MappingLookup mappingL context.sourceToParse().source(), context.sourceToParse().getXContentType(), dynamicUpdate, - meteringParserDecorator.meteredDocumentSize() + meteringParserDecorator.meteredDocumentSize(), + context.getTsid() ) { @Override public String documentDescription() { @@ -1072,7 +1073,7 @@ private static class RootDocumentParserContext extends DocumentParserContext { private final long maxAllowedNumNestedDocs; private long numNestedDocs; private boolean docsReversed = false; - private final BytesRef tsid; + private BytesRef tsid; RootDocumentParserContext( MappingLookup mappingLookup, @@ -1092,8 +1093,9 @@ private static class RootDocumentParserContext extends DocumentParserContext { if (tsid == null && indexSettings.getMode() == IndexMode.TIME_SERIES && indexSettings.getIndexRouting() instanceof IndexRouting.ExtractFromSource.ForIndexDimensions forIndexDimensions) { - // the tsid is normally set on the coordinating node during shard routing and passed to the data node via the index request - // but when applying a translog operation, shard routing is not happening, and we have to create the tsid from source + // The tsid is normally set on the coordinating node during shard routing and passed to the data node via the index request. + // When applying a translog operation from an older version that didn't include the tsid in the translog, yet, + // we have to re-create the tsid from source. tsid = forIndexDimensions.buildTsid(source.getXContentType(), source.source()); } this.tsid = tsid; @@ -1158,6 +1160,9 @@ protected void addDoc(LuceneDocument doc) { @Override public BytesRef getTsid() { + if (tsid == null && getRoutingFields() instanceof RoutingPathFields routingPathFields) { + tsid = routingPathFields.buildHash().toBytesRef(); + } return this.tsid; } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DocumentParserContext.java b/server/src/main/java/org/elasticsearch/index/mapper/DocumentParserContext.java index 0b1a64713857a..606c9006ff115 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DocumentParserContext.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DocumentParserContext.java @@ -271,7 +271,7 @@ protected DocumentParserContext( null, null, SeqNoFieldMapper.SequenceIDFields.emptySeqID(mappingParserContext.getIndexSettings().seqNoIndexOptions()), - RoutingFields.fromIndexSettings(mappingParserContext.getIndexSettings()), + RoutingFields.fromIndexSettings(mappingParserContext.getIndexSettings(), source), parent, dynamic, new HashSet<>(), @@ -871,6 +871,10 @@ public final MapperBuilderContext createDynamicMapperBuilderContext() { protected abstract void addDoc(LuceneDocument doc); + /** + * Gets or creates the time series id for this document, or null if the index is not a time series index. + * This method must only be called after document parsing has completed, so that all dimension fields have been extracted. + */ @Nullable public abstract BytesRef getTsid(); diff --git a/server/src/main/java/org/elasticsearch/index/mapper/ParsedDocument.java b/server/src/main/java/org/elasticsearch/index/mapper/ParsedDocument.java index 72fd812d982d8..392d411d74958 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/ParsedDocument.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/ParsedDocument.java @@ -14,6 +14,7 @@ import org.apache.lucene.util.BytesRef; import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.core.Nullable; import org.elasticsearch.index.mapper.MapperService.MergeReason; import org.elasticsearch.plugins.internal.XContentMeteringParserDecorator; import org.elasticsearch.xcontent.XContentType; @@ -36,6 +37,8 @@ public class ParsedDocument { private final List documents; private final long normalizedSize; + @Nullable + private final BytesRef tsid; private BytesReference source; private XContentType xContentType; @@ -63,7 +66,8 @@ public static ParsedDocument noopTombstone(SeqNoFieldMapper.SeqNoIndexOptions se new BytesArray("{}"), XContentType.JSON, null, - XContentMeteringParserDecorator.UNKNOWN_SIZE + XContentMeteringParserDecorator.UNKNOWN_SIZE, + null ); } @@ -88,7 +92,8 @@ public static ParsedDocument deleteTombstone(SeqNoFieldMapper.SeqNoIndexOptions new BytesArray("{}"), XContentType.JSON, null, - XContentMeteringParserDecorator.UNKNOWN_SIZE + XContentMeteringParserDecorator.UNKNOWN_SIZE, + null ); } @@ -101,7 +106,8 @@ public ParsedDocument( BytesReference source, XContentType xContentType, Mapping dynamicMappingsUpdate, - long normalizedSize + long normalizedSize, + @Nullable BytesRef tsid ) { this.version = version; this.seqID = seqID; @@ -112,6 +118,7 @@ public ParsedDocument( this.dynamicMappingsUpdate = dynamicMappingsUpdate; this.xContentType = xContentType; this.normalizedSize = normalizedSize; + this.tsid = tsid; } public String id() { @@ -150,6 +157,10 @@ public XContentType getXContentType() { return this.xContentType; } + public @Nullable BytesRef tsid() { + return tsid; + } + public void setSource(BytesReference source, XContentType xContentType) { this.source = source; this.xContentType = xContentType; diff --git a/server/src/main/java/org/elasticsearch/index/mapper/RoutingFields.java b/server/src/main/java/org/elasticsearch/index/mapper/RoutingFields.java index 304a57bc647a4..0ad4f4e0b49aa 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/RoutingFields.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/RoutingFields.java @@ -22,8 +22,8 @@ public interface RoutingFields { /** * Collect routing fields from index settings */ - static RoutingFields fromIndexSettings(IndexSettings indexSettings) { - return indexSettings.getMode().buildRoutingFields(indexSettings); + static RoutingFields fromIndexSettings(IndexSettings indexSettings, SourceToParse source) { + return indexSettings.getMode().buildRoutingFields(indexSettings, source); } /** diff --git a/server/src/main/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapper.java index 288ddc116a08e..a3c1696f8028a 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapper.java @@ -153,21 +153,24 @@ public void postParse(DocumentParserContext context) throws IOException { final BytesRef timeSeriesId; final RoutingPathFields routingPathFields; + if (context.getRoutingFields() instanceof RoutingPathFields routingPathFieldsFromContext) { + routingPathFields = routingPathFieldsFromContext; + } else { + routingPathFields = null; + } if (getIndexVersionCreated(context).before(IndexVersions.TIME_SERIES_ID_HASHING)) { - routingPathFields = (RoutingPathFields) context.getRoutingFields(); + assert routingPathFields != null : "routing path fields are required for legacy indices"; long limit = context.indexSettings().getValue(MapperService.INDEX_MAPPING_DIMENSION_FIELDS_LIMIT_SETTING); int size = routingPathFields.routingValues().size(); if (size > limit) { throw new MapperException("Too many dimension fields [" + size + "], max [" + limit + "] dimension fields allowed"); } timeSeriesId = buildLegacyTsid(routingPathFields).toBytesRef(); - } else if (context.getRoutingFields() instanceof RoutingPathFields routingPathFieldsFromContext) { - routingPathFields = routingPathFieldsFromContext; - timeSeriesId = routingPathFields.buildHash().toBytesRef(); } else { - routingPathFields = null; - assert context.getTsid() != null; timeSeriesId = context.getTsid(); + if (timeSeriesId == null) { + throw new IllegalStateException("tsid is missing from the context" + context.documentDescription()); + } } if (this.useDocValuesSkipper) { diff --git a/server/src/main/java/org/elasticsearch/index/shard/IndexShard.java b/server/src/main/java/org/elasticsearch/index/shard/IndexShard.java index e94ef25aa7822..9d9973e02014f 100644 --- a/server/src/main/java/org/elasticsearch/index/shard/IndexShard.java +++ b/server/src/main/java/org/elasticsearch/index/shard/IndexShard.java @@ -2130,7 +2130,14 @@ private Engine.Result applyTranslogOperation(Engine engine, Translog.Operation o index.getAutoGeneratedIdTimestamp(), true, origin, - new SourceToParse(index.id(), index.source(), XContentHelper.xContentType(index.source()), index.routing()) + new SourceToParse( + index.id(), + index.source(), + XContentHelper.xContentType(index.source()), + index.routing(), + Map.of(), + index.tsid() + ) ); } case DELETE -> { diff --git a/server/src/main/java/org/elasticsearch/index/translog/Translog.java b/server/src/main/java/org/elasticsearch/index/translog/Translog.java index 6e83a684cfa82..884ee4b017d00 100644 --- a/server/src/main/java/org/elasticsearch/index/translog/Translog.java +++ b/server/src/main/java/org/elasticsearch/index/translog/Translog.java @@ -10,6 +10,7 @@ package org.elasticsearch.index.translog; import org.apache.lucene.store.AlreadyClosedException; +import org.apache.lucene.util.BytesRef; import org.elasticsearch.TransportVersions; import org.elasticsearch.common.Strings; import org.elasticsearch.common.UUIDs; @@ -31,6 +32,7 @@ import org.elasticsearch.index.engine.TranslogOperationAsserter; import org.elasticsearch.index.mapper.IdFieldMapper; import org.elasticsearch.index.mapper.MapperService; +import org.elasticsearch.index.mapper.TimeSeriesIdFieldMapper; import org.elasticsearch.index.mapper.Uid; import org.elasticsearch.index.seqno.SequenceNumbers; import org.elasticsearch.index.shard.AbstractIndexShardComponent; @@ -1164,13 +1166,15 @@ public static final class Index extends Operation { public static final int FORMAT_NO_PARENT = 9; // since 7.0 public static final int FORMAT_NO_VERSION_TYPE = FORMAT_NO_PARENT + 1; public static final int FORMAT_NO_DOC_TYPE = FORMAT_NO_VERSION_TYPE + 1; - public static final int SERIALIZATION_FORMAT = FORMAT_NO_DOC_TYPE; + public static final int FORMAT_TSID = FORMAT_NO_DOC_TYPE + 1; + public static final int SERIALIZATION_FORMAT = FORMAT_TSID; private final String id; private final long autoGeneratedIdTimestamp; private final long version; private final BytesReference source; private final String routing; + private final BytesRef tsid; private static Index readFrom(StreamInput in) throws IOException { final int format = in.readVInt(); // SERIALIZATION_FORMAT @@ -1189,7 +1193,13 @@ private static Index readFrom(StreamInput in) throws IOException { long autoGeneratedIdTimestamp = in.readLong(); long seqNo = in.readLong(); long primaryTerm = in.readLong(); - return new Index(id, seqNo, primaryTerm, version, source, routing, autoGeneratedIdTimestamp); + BytesRef tsid; + if (format >= FORMAT_TSID) { + tsid = in.readOptionalBytesRef(); + } else { + tsid = null; + } + return new Index(id, seqNo, primaryTerm, version, source, routing, autoGeneratedIdTimestamp, tsid); } public Index(Engine.Index index, Engine.IndexResult indexResult) { @@ -1200,7 +1210,8 @@ public Index(Engine.Index index, Engine.IndexResult indexResult) { indexResult.getVersion(), index.source(), index.routing(), - index.getAutoGeneratedIdTimestamp() + index.getAutoGeneratedIdTimestamp(), + index.tsid() ); } @@ -1211,7 +1222,8 @@ public Index( long version, BytesReference source, String routing, - long autoGeneratedIdTimestamp + long autoGeneratedIdTimestamp, + @Nullable BytesRef tsid ) { super(seqNo, primaryTerm); this.id = id; @@ -1219,6 +1231,7 @@ public Index( this.version = version; this.routing = routing; this.autoGeneratedIdTimestamp = autoGeneratedIdTimestamp; + this.tsid = tsid; } @Override @@ -1228,10 +1241,8 @@ public Type opType() { @Override public long estimateSize() { - return (2 * id.length()) + source.length() + (routing != null ? 2 * routing.length() : 0) + (4 * Long.BYTES); // timestamp, - // seq_no, - // primary_term, - // and version + return (2 * id.length()) + source.length() + (routing != null ? 2 * routing.length() : 0) + (tsid == null ? 0 : tsid.length) + + (4 * Long.BYTES); // timestamp, seq_no, primary_term, and version } public String id() { @@ -1250,6 +1261,11 @@ public long version() { return this.version; } + @Nullable + public BytesRef tsid() { + return this.tsid; + } + @Override public void writeBody(final StreamOutput out) throws IOException { final int format = out.getTransportVersion().onOrAfter(TransportVersions.V_8_0_0) @@ -1266,6 +1282,9 @@ public void writeBody(final StreamOutput out) throws IOException { out.writeLong(autoGeneratedIdTimestamp); out.writeLong(seqNo); out.writeLong(primaryTerm); + if (format >= FORMAT_TSID) { + out.writeBytesRef(tsid); + } } @Override @@ -1290,6 +1309,7 @@ public int hashCode() { result = 31 * result + source.hashCode(); result = 31 * result + (routing != null ? routing.hashCode() : 0); result = 31 * result + Long.hashCode(autoGeneratedIdTimestamp); + result = 31 * result + (tsid != null ? tsid.hashCode() : 0); return result; } @@ -1307,6 +1327,8 @@ public String toString() { + version + ", autoGeneratedIdTimestamp=" + autoGeneratedIdTimestamp + + ", tsid=" + + (tsid != null ? TimeSeriesIdFieldMapper.encodeTsid(tsid) : null) + '}'; } @@ -1319,7 +1341,8 @@ public static boolean equalsWithoutAutoGeneratedTimestamp(Translog.Index o1, Tra || o1.seqNo != o2.seqNo || o1.primaryTerm != o2.primaryTerm || o1.id.equals(o2.id) == false - || Objects.equals(o1.routing, o2.routing) == false) { + || Objects.equals(o1.routing, o2.routing) == false + || Objects.equals(o1.tsid, o2.tsid) == false) { return false; } diff --git a/server/src/test/java/org/elasticsearch/action/resync/ResyncReplicationRequestTests.java b/server/src/test/java/org/elasticsearch/action/resync/ResyncReplicationRequestTests.java index aa55c108d37d7..a265405950ebb 100644 --- a/server/src/test/java/org/elasticsearch/action/resync/ResyncReplicationRequestTests.java +++ b/server/src/test/java/org/elasticsearch/action/resync/ResyncReplicationRequestTests.java @@ -33,7 +33,8 @@ public void testSerialization() throws IOException { randomNonNegativeLong(), new BytesArray(bytes), randomBoolean() ? randomAlphaOfLengthBetween(1, 5) : null, - randomNonNegativeLong() + randomNonNegativeLong(), + randomBoolean() ? randomBytesReference(between(16, 32)).toBytesRef() : null ); final ShardId shardId = new ShardId(new Index("index", "uuid"), 0); final ResyncReplicationRequest before = new ResyncReplicationRequest(shardId, 42L, 100, new Translog.Operation[] { index }); diff --git a/server/src/test/java/org/elasticsearch/index/IndexingSlowLogTests.java b/server/src/test/java/org/elasticsearch/index/IndexingSlowLogTests.java index a08029c7fb3b4..f8d9c9f483c6a 100644 --- a/server/src/test/java/org/elasticsearch/index/IndexingSlowLogTests.java +++ b/server/src/test/java/org/elasticsearch/index/IndexingSlowLogTests.java @@ -217,7 +217,8 @@ public void testSlowLogMessageHasJsonFields() throws IOException { source, XContentType.JSON, null, - XContentMeteringParserDecorator.UNKNOWN_SIZE + XContentMeteringParserDecorator.UNKNOWN_SIZE, + null ); Index index = new Index("foo", "123"); // Turning off document logging doesn't log source[] @@ -246,7 +247,8 @@ public void testSlowLogMessageHasAdditionalFields() throws IOException { source, XContentType.JSON, null, - XContentMeteringParserDecorator.UNKNOWN_SIZE + XContentMeteringParserDecorator.UNKNOWN_SIZE, + null ); Index index = new Index("foo", "123"); // Turning off document logging doesn't log source[] @@ -276,7 +278,8 @@ public void testEmptyRoutingField() throws IOException { source, XContentType.JSON, null, - XContentMeteringParserDecorator.UNKNOWN_SIZE + XContentMeteringParserDecorator.UNKNOWN_SIZE, + null ); Index index = new Index("foo", "123"); @@ -295,7 +298,8 @@ public void testSlowLogParsedDocumentPrinterSourceToLog() throws IOException { source, XContentType.JSON, null, - XContentMeteringParserDecorator.UNKNOWN_SIZE + XContentMeteringParserDecorator.UNKNOWN_SIZE, + null ); Index index = new Index("foo", "123"); // Turning off document logging doesn't log source[] @@ -327,7 +331,8 @@ public void testSlowLogParsedDocumentPrinterSourceToLog() throws IOException { source, XContentType.JSON, null, - XContentMeteringParserDecorator.UNKNOWN_SIZE + XContentMeteringParserDecorator.UNKNOWN_SIZE, + null ); final XContentParseException e = expectThrows( diff --git a/server/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java b/server/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java index cfce840cda9f0..3cfc0402ede22 100644 --- a/server/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java +++ b/server/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java @@ -5537,7 +5537,8 @@ public void testSeqNoGenerator() throws IOException { source, XContentType.JSON, null, - XContentMeteringParserDecorator.UNKNOWN_SIZE + XContentMeteringParserDecorator.UNKNOWN_SIZE, + randomBoolean() ? randomBytesReference(between(16, 32)).toBytesRef() : null ); final Engine.Index index = new Engine.Index( diff --git a/server/src/test/java/org/elasticsearch/index/engine/TranslogOperationAsserterTests.java b/server/src/test/java/org/elasticsearch/index/engine/TranslogOperationAsserterTests.java index d0455c14bd784..9345c2f67afb0 100644 --- a/server/src/test/java/org/elasticsearch/index/engine/TranslogOperationAsserterTests.java +++ b/server/src/test/java/org/elasticsearch/index/engine/TranslogOperationAsserterTests.java @@ -47,7 +47,8 @@ Translog.Index toIndexOp(String source) throws IOException { 1, new BytesArray(Strings.toString(builder)), null, - IndexRequest.UNSET_AUTO_GENERATED_TIMESTAMP + IndexRequest.UNSET_AUTO_GENERATED_TIMESTAMP, + null ); } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/TsidExtractingIdFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/TsidExtractingIdFieldMapperTests.java index 85bd9950ba99c..6d04936c5ee8e 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/TsidExtractingIdFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/TsidExtractingIdFieldMapperTests.java @@ -11,7 +11,9 @@ import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; import org.apache.lucene.index.IndexableField; +import org.apache.lucene.util.BytesRef; import org.elasticsearch.cluster.metadata.IndexMetadata; +import org.elasticsearch.common.Strings; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.ByteUtils; @@ -28,6 +30,7 @@ import java.util.ArrayList; import java.util.Base64; import java.util.List; +import java.util.Map; import java.util.stream.Stream; import static org.hamcrest.Matchers.equalTo; @@ -775,18 +778,64 @@ public void testExpectedIdWithIndexDimensions() throws IOException { public void testProvideExpectedIdWithRoutingPath() throws IOException { assertThat( - parse(testCase.expectedIdWithRoutingPath, mapperService(false), testCase.source).id(), + parse(testCase.expectedIdWithRoutingPath, null, mapperService(false), testCase.source).id(), equalTo(testCase.expectedIdWithRoutingPath) ); } public void testProvideExpectedIdWithIndexDimensions() throws IOException { assertThat( - parse(testCase.expectedIdWithIndexDimensions, mapperService(true), testCase.source).id(), + parse(testCase.expectedIdWithIndexDimensions, null, mapperService(true), testCase.source).id(), equalTo(testCase.expectedIdWithIndexDimensions) ); } + public void testProvideExpectedTsidWithRoutingPath() throws IOException { + assertThat( + parse(null, testCase.expectedTsidWithRoutingPath, mapperService(false), testCase.source).id(), + equalTo(testCase.expectedIdWithRoutingPath) + ); + } + + public void testProvideExpectedTsidWithIndexDimensions() throws IOException { + assertThat( + parse(null, testCase.expectedTsidWithIndexDimensions, mapperService(true), testCase.source).id(), + equalTo(testCase.expectedIdWithIndexDimensions) + ); + } + + public void testProvideExpectedIdAndTsidWithRoutingPath() throws IOException { + assertThat( + parse(testCase.expectedIdWithRoutingPath, testCase.expectedTsidWithRoutingPath, mapperService(false), testCase.source).id(), + equalTo(testCase.expectedIdWithRoutingPath) + ); + } + + public void testProvideExpectedIdAndTsidWithIndexDimensions() throws IOException { + assertThat( + parse(testCase.expectedIdWithIndexDimensions, testCase.expectedTsidWithIndexDimensions, mapperService(true), testCase.source) + .id(), + equalTo(testCase.expectedIdWithIndexDimensions) + ); + } + + public void testProvideExternalTsid() throws IOException { + // When replaying translog operations, both the id and tsid are provided from the translog entry. + // We must not recompute the id and the tsid from the document source and instead use the provided ones. + // Here we generate a random tsid that is different from the expected one and verify that the provided id is returned unchanged + byte[] tsidBytes = randomByteArrayOfLength(between(16, 32)); + String tsid = Strings.BASE_64_NO_PADDING_URL_ENCODER.encodeToString(tsidBytes); + String id = TsidExtractingIdFieldMapper.createId( + ROUTING_HASH, + new BytesRef(tsidBytes), + DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.parseMillis(testCase.expectedTimestamp) + ); + + // we test with both tsid creation strategies - index.dimensions and index.routing_path + assertThat(parse(id, tsid, mapperService(true), testCase.source).id(), equalTo(id)); + assertThat(parse(id, tsid, mapperService(false), testCase.source).id(), equalTo(id)); + } + public void testEquivalentSourcesWithRoutingPath() throws IOException { MapperService mapperService = mapperService(false); for (CheckedConsumer equivalent : testCase.equivalentSources) { @@ -795,11 +844,15 @@ public void testEquivalentSourcesWithRoutingPath() throws IOException { } private ParsedDocument parse(MapperService mapperService, CheckedConsumer source) throws IOException { - return parse(null, mapperService, source); + return parse(null, null, mapperService, source); } - private ParsedDocument parse(@Nullable String id, MapperService mapperService, CheckedConsumer source) - throws IOException { + private ParsedDocument parse( + @Nullable String id, + String tsid, + MapperService mapperService, + CheckedConsumer source + ) throws IOException { try (XContentBuilder builder = XContentBuilder.builder(randomFrom(XContentType.values()).xContent())) { builder.startObject(); source.accept(builder); @@ -808,7 +861,9 @@ private ParsedDocument parse(@Nullable String id, MapperService mapperService, C id, BytesReference.bytes(builder), builder.contentType(), - TimeSeriesRoutingHashFieldMapper.encode(ROUTING_HASH) + TimeSeriesRoutingHashFieldMapper.encode(ROUTING_HASH), + Map.of(), + tsid != null ? new BytesRef(Base64.getUrlDecoder().decode(tsid)) : null ); return mapperService.documentParser().parseDocument(sourceToParse, mapperService.mappingLookup()); } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/vectors/SyntheticVectorFieldsRecoveryTests.java b/server/src/test/java/org/elasticsearch/index/mapper/vectors/SyntheticVectorFieldsRecoveryTests.java index 3718dae1c7d3e..46b28d29af8d2 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/vectors/SyntheticVectorFieldsRecoveryTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/vectors/SyntheticVectorFieldsRecoveryTests.java @@ -148,7 +148,8 @@ public void testSnapshotRecovery() throws IOException { result.getVersion(), op.source(), op.routing(), - op.getAutoGeneratedIdTimestamp() + op.getAutoGeneratedIdTimestamp(), + op.tsid() ) ); diff --git a/server/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java b/server/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java index 6a41c39e0bb99..d77000a944277 100644 --- a/server/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java +++ b/server/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java @@ -29,6 +29,7 @@ import org.apache.lucene.util.Constants; import org.elasticsearch.ElasticsearchException; import org.elasticsearch.ExceptionsHelper; +import org.elasticsearch.TransportVersion; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.admin.indices.flush.FlushRequest; import org.elasticsearch.action.admin.indices.forcemerge.ForceMergeRequest; @@ -43,6 +44,7 @@ import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.cluster.node.DiscoveryNodeUtils; import org.elasticsearch.cluster.routing.AllocationId; +import org.elasticsearch.cluster.routing.IndexRouting; import org.elasticsearch.cluster.routing.IndexShardRoutingTable; import org.elasticsearch.cluster.routing.RecoverySource; import org.elasticsearch.cluster.routing.ShardRouting; @@ -66,6 +68,7 @@ import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.common.util.concurrent.ReleasableLock; import org.elasticsearch.common.util.concurrent.RunOnce; +import org.elasticsearch.common.xcontent.XContentHelper; import org.elasticsearch.core.Assertions; import org.elasticsearch.core.CheckedFunction; import org.elasticsearch.core.IOUtils; @@ -77,6 +80,7 @@ import org.elasticsearch.index.IndexModule; import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.IndexVersion; +import org.elasticsearch.index.VersionType; import org.elasticsearch.index.codec.CodecService; import org.elasticsearch.index.engine.CommitStats; import org.elasticsearch.index.engine.DocIdSeqNoAndSource; @@ -130,6 +134,7 @@ import org.elasticsearch.test.DummyShardLock; import org.elasticsearch.test.FieldMaskingReader; import org.elasticsearch.test.MockLog; +import org.elasticsearch.test.TransportVersionUtils; import org.elasticsearch.test.junit.annotations.TestIssueLogging; import org.elasticsearch.test.junit.annotations.TestLogging; import org.elasticsearch.test.store.MockFSDirectoryFactory; @@ -3147,7 +3152,8 @@ public void testRecoverFromTranslog() throws IOException { 1, new BytesArray("{\"foo\" : \"bar\"}".getBytes(StandardCharsets.UTF_8)), null, - -1 + -1, + null ) ); } else { @@ -3160,7 +3166,8 @@ public void testRecoverFromTranslog() throws IOException { 1, new BytesArray("{\"foo\" : \"bar}".getBytes(StandardCharsets.UTF_8)), null, - -1 + -1, + null ) ); numCorruptEntries++; @@ -3187,6 +3194,160 @@ public void testRecoverFromTranslog() throws IOException { closeShards(primary); } + public void testRecoverFromTranslogWhenDimensionsChange() throws IOException { + // Prepare an index with a single time series dimension and a non-dynamic mapping. + boolean indexDimensions = randomBoolean(); + Settings settings = indexSettings(IndexVersion.current(), 1, 1).put("index.mode", "time_series") + .put("index.time_series.start_time", "2025-01-01T00:00:00") + .put("index.time_series.end_time", "2025-01-02T00:00:00") + .put(indexDimensions ? "index.dimensions" : "index.routing_path", "dim1") + .build(); + IndexMetadata metadata = IndexMetadata.builder("test").putMapping(""" + { + "_data_stream_timestamp": { + "enabled": true + }, + "dynamic": false, + "properties": { + "@timestamp": { + "type": "date" + }, + "dim1": { + "type": "keyword", + "time_series_dimension": "true" + } + } + } + """).settings(settings).primaryTerm(0, randomLongBetween(1, Long.MAX_VALUE)).build(); + + // Index a document that has a field for the single dimension and another field that is not in the mappings (yet). + // As the mappings are non-dynamic this field will be ignored. + // We'll test how the recovery behaves when this field is later added as a second dimension. + IndexRequest indexRequest = new IndexRequest().source(Map.of("@timestamp", "2025", "dim1", "foo", "dim2", "bar")); + IndexRouting indexRouting = IndexRouting.fromIndexMetadata(metadata); + indexRouting.indexShard(indexRequest); + indexRouting.postProcess(indexRequest); + + IndexShard primary = newShard(new ShardId(metadata.getIndex(), 0), true, "n1", metadata, null); + Engine.Index index = prepareIndex(primary, indexRequest); + assertThat(index.id(), not(nullValue())); + assertThat(index.tsid(), not(nullValue())); + + // Simulate a mappings change where a second dimension is added that matches the ignored field in the original index request. + if (indexDimensions) { + settings = Settings.builder().put(settings).put("index.dimensions", "dim1,dim2").build(); + } + IndexMetadata withAdditionalDimension = IndexMetadata.builder(metadata).settings(settings).putMapping(""" + { + "_data_stream_timestamp": { + "enabled": true + }, + "dynamic": false, + "properties": { + "@timestamp": { + "type": "date" + }, + "dim1": { + "type": "keyword", + "time_series_dimension": "true" + }, + "dim2": { + "type": "keyword", + "time_series_dimension": "true" + } + } + } + """).build(); + + // Creates a replica shard with the updated mappings and recovers the translog index operation on it. + IndexShard replica = newShard(new ShardId(withAdditionalDimension.getIndex(), 0), false, "n1", withAdditionalDimension, null); + Engine.IndexResult indexResult = new Engine.IndexResult(1, index.primaryTerm(), index.seqNo(), true, index.id()); + Translog.Index translogIndex = copy(new Translog.Index(index, indexResult)); + Engine.Index recoveryIndex = prepareIndex(replica, translogIndex); + + // The index operation that is recovered from translog should have the same id and tsid as the original index operation, + // despite the mappings having changed in the meantime. + // Otherwise, the primary and replica would diverge. + assertThat(recoveryIndex, not(sameInstance(index))); + assertThat(recoveryIndex.id(), equalTo(index.id())); + assertThat(recoveryIndex.tsid(), equalTo(index.tsid())); + + closeShards(primary); + closeShards(replica); + } + + private static Translog.Index copy(Translog.Index index) throws IOException { + TransportVersion wireVersion = TransportVersionUtils.randomVersion(); + BytesStreamOutput out = new BytesStreamOutput(); + out.setTransportVersion(wireVersion); + index.writeTo(out); + StreamInput in = out.bytes().streamInput(); + in.setTransportVersion(wireVersion); + Translog.Index copy = (Translog.Index) Translog.Operation.readOperation(in); + assertThat(index, equalTo(copy)); + assertThat(Translog.Index.equalsWithoutAutoGeneratedTimestamp(index, copy, true), equalTo(true)); + assertThat(index.hashCode(), equalTo(copy.hashCode())); + assertThat(index.estimateSize(), equalTo(copy.estimateSize())); + assertThat(index.toString(), equalTo(copy.toString())); + return copy; + } + + private static Engine.Index prepareIndex(IndexShard primary, Translog.Index index) { + return IndexShard.prepareIndex( + primary.mapperService(), + createSourceToParse(index), + randomNonNegativeLong(), + primary.getPendingPrimaryTerm(), + index.version(), + VersionType.INTERNAL, + Engine.Operation.Origin.PRIMARY, + index.getAutoGeneratedIdTimestamp(), + true, + UNASSIGNED_SEQ_NO, + 0, + primary.getRelativeTimeInNanos() + ); + } + + private static Engine.Index prepareIndex(IndexShard primary, IndexRequest indexRequest) { + return IndexShard.prepareIndex( + primary.mapperService(), + createSourceToParse(indexRequest), + randomNonNegativeLong(), + primary.getPendingPrimaryTerm(), + indexRequest.version(), + VersionType.INTERNAL, + Engine.Operation.Origin.PRIMARY, + indexRequest.getAutoGeneratedTimestamp(), + indexRequest.isRetry(), + indexRequest.ifSeqNo(), + indexRequest.ifPrimaryTerm(), + primary.getRelativeTimeInNanos() + ); + } + + private static SourceToParse createSourceToParse(IndexRequest indexRequest) { + return new SourceToParse( + indexRequest.id(), + indexRequest.source(), + indexRequest.getContentType(), + indexRequest.routing(), + indexRequest.getDynamicTemplates(), + indexRequest.tsid() + ); + } + + private static SourceToParse createSourceToParse(Translog.Index index) { + return new SourceToParse( + index.id(), + index.source(), + XContentHelper.xContentType(index.source()), + index.routing(), + Map.of(), + index.tsid() + ); + } + public void testShardActiveDuringInternalRecovery() throws IOException { boolean isPrimary = randomBoolean(); IndexShard shard = newStartedShard(isPrimary); @@ -4828,7 +4989,8 @@ public void testResetEngineWithBrokenTranslog() throws Exception { 1, new BytesArray("{\"foo\" : \"bar\"}".getBytes(StandardCharsets.UTF_8)), null, - -1 + -1, + null ) ), // entries with corrupted source @@ -4841,7 +5003,8 @@ public void testResetEngineWithBrokenTranslog() throws Exception { 1, new BytesArray("{\"foo\" : \"bar}".getBytes(StandardCharsets.UTF_8)), null, - -1 + -1, + null ) ) ).collect(Collectors.toCollection(ArrayList::new)); diff --git a/server/src/test/java/org/elasticsearch/index/shard/RefreshListenersTests.java b/server/src/test/java/org/elasticsearch/index/shard/RefreshListenersTests.java index bf6a4f4ec2d56..1a0b96a69dde4 100644 --- a/server/src/test/java/org/elasticsearch/index/shard/RefreshListenersTests.java +++ b/server/src/test/java/org/elasticsearch/index/shard/RefreshListenersTests.java @@ -591,7 +591,8 @@ private Engine.IndexResult index(String id, String testFieldValue) throws IOExce source, XContentType.JSON, null, - XContentMeteringParserDecorator.UNKNOWN_SIZE + XContentMeteringParserDecorator.UNKNOWN_SIZE, + null ); Engine.Index index = new Engine.Index(uid, engine.config().getPrimaryTermSupplier().getAsLong(), doc); return engine.index(index); diff --git a/server/src/test/java/org/elasticsearch/index/shard/ShardGetServiceTests.java b/server/src/test/java/org/elasticsearch/index/shard/ShardGetServiceTests.java index a10a23db0b838..4e9f2a15e9c15 100644 --- a/server/src/test/java/org/elasticsearch/index/shard/ShardGetServiceTests.java +++ b/server/src/test/java/org/elasticsearch/index/shard/ShardGetServiceTests.java @@ -422,7 +422,8 @@ Translog.Index toIndexOp(String source) throws IOException { 1, new BytesArray(org.elasticsearch.common.Strings.toString(builder)), null, - IndexRequest.UNSET_AUTO_GENERATED_TIMESTAMP + IndexRequest.UNSET_AUTO_GENERATED_TIMESTAMP, + null ); } } diff --git a/server/src/test/java/org/elasticsearch/index/translog/TranslogTests.java b/server/src/test/java/org/elasticsearch/index/translog/TranslogTests.java index 8032eda2dba1a..1532420b90d64 100644 --- a/server/src/test/java/org/elasticsearch/index/translog/TranslogTests.java +++ b/server/src/test/java/org/elasticsearch/index/translog/TranslogTests.java @@ -3433,6 +3433,7 @@ public void testTranslogOpSerialization() throws Exception { document.add(idField); document.add(versionField); seqID.addFields(document); + BytesRef tsid = randomBoolean() ? null : new BytesRef(randomByteArrayOfLength(between(16, 32))); ParsedDocument doc = new ParsedDocument( versionField, seqID, @@ -3442,7 +3443,8 @@ public void testTranslogOpSerialization() throws Exception { B_1, XContentType.JSON, null, - XContentMeteringParserDecorator.UNKNOWN_SIZE + XContentMeteringParserDecorator.UNKNOWN_SIZE, + tsid ); Engine.Index eIndex = new Engine.Index( @@ -3474,6 +3476,7 @@ public void testTranslogOpSerialization() throws Exception { in.setTransportVersion(wireVersion); Translog.Index serializedIndex = (Translog.Index) Translog.Operation.readOperation(in); assertEquals(index, serializedIndex); + assertEquals(index.tsid(), tsid); Engine.Delete eDelete = new Engine.Delete( doc.id(), diff --git a/server/src/test/java/org/elasticsearch/indices/recovery/RecoverySourceHandlerTests.java b/server/src/test/java/org/elasticsearch/indices/recovery/RecoverySourceHandlerTests.java index 13d3e8360e434..b33d92480c359 100644 --- a/server/src/test/java/org/elasticsearch/indices/recovery/RecoverySourceHandlerTests.java +++ b/server/src/test/java/org/elasticsearch/indices/recovery/RecoverySourceHandlerTests.java @@ -256,7 +256,7 @@ public StartRecoveryRequest getStartRecoveryRequest() { } public void testSendSnapshotSendsOps() throws IOException { - IndexOpFactory iof = randomBoolean() ? new StandardModeIndexOpFactory() : new TimeSeriesModeIndexOpFactory(); + IndexOpFactory iof = false ? new StandardModeIndexOpFactory() : new TimeSeriesModeIndexOpFactory(); final int fileChunkSizeInBytes = between(1, 4096); final StartRecoveryRequest request = getStartRecoveryRequest(); final IndexShard shard = mock(IndexShard.class); @@ -1934,7 +1934,8 @@ public static Translog.Operation generateOperation(long seqNo) { randomNonNegativeLong(), TRANSLOG_OPERATION_SOURCE, randomBoolean() ? randomAlphaOfLengthBetween(1, 5) : null, - randomNonNegativeLong() + randomNonNegativeLong(), + randomBoolean() ? randomBytesReference(between(16, 32)).toBytesRef() : null ); } else if (randomBoolean()) { op = new Translog.Delete("id", seqNo, randomNonNegativeLong(), randomNonNegativeLong()); diff --git a/test/framework/src/main/java/org/elasticsearch/index/engine/EngineTestCase.java b/test/framework/src/main/java/org/elasticsearch/index/engine/EngineTestCase.java index e552bb133add4..b2373177d54d6 100644 --- a/test/framework/src/main/java/org/elasticsearch/index/engine/EngineTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/index/engine/EngineTestCase.java @@ -486,7 +486,8 @@ protected static ParsedDocument testParsedDocument( source, XContentType.JSON, mappingUpdate, - XContentMeteringParserDecorator.UNKNOWN_SIZE + XContentMeteringParserDecorator.UNKNOWN_SIZE, + null ); } diff --git a/test/framework/src/main/java/org/elasticsearch/index/translog/TranslogOperationsUtils.java b/test/framework/src/main/java/org/elasticsearch/index/translog/TranslogOperationsUtils.java index a74199234476f..64c204bdb561b 100644 --- a/test/framework/src/main/java/org/elasticsearch/index/translog/TranslogOperationsUtils.java +++ b/test/framework/src/main/java/org/elasticsearch/index/translog/TranslogOperationsUtils.java @@ -39,7 +39,8 @@ public static Translog.Index indexOp(String id, long seqNo, long primaryTerm, St Versions.MATCH_ANY, new BytesArray(Objects.requireNonNull(source).getBytes(StandardCharsets.UTF_8)), null, - -1L + -1L, + null ); } } diff --git a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/bulk/TransportBulkShardOperationsAction.java b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/bulk/TransportBulkShardOperationsAction.java index 9a6e187e5544e..df38df7ef8234 100644 --- a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/bulk/TransportBulkShardOperationsAction.java +++ b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/bulk/TransportBulkShardOperationsAction.java @@ -144,7 +144,8 @@ public static Translog.Operation rewriteOperationWithPrimaryTerm(Translog.Operat index.version(), index.source(), index.routing(), - index.getAutoGeneratedIdTimestamp() + index.getAutoGeneratedIdTimestamp(), + index.tsid() ); } case DELETE -> { diff --git a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/ShardFollowTaskReplicationTests.java b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/ShardFollowTaskReplicationTests.java index d0bf7f041db85..ac7b7d7345098 100644 --- a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/ShardFollowTaskReplicationTests.java +++ b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/ShardFollowTaskReplicationTests.java @@ -400,7 +400,7 @@ public void testAddNewFollowingReplica() throws Exception { final int numDocs = between(1, 100); final List operations = new ArrayList<>(numDocs); for (int i = 0; i < numDocs; i++) { - operations.add(new Translog.Index(Integer.toString(i), i, primaryTerm, 0, source, null, -1)); + operations.add(new Translog.Index(Integer.toString(i), i, primaryTerm, 0, source, null, -1, null)); } Future recoveryFuture = null; Settings settings = Settings.builder() diff --git a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/bulk/BulkShardOperationsTests.java b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/bulk/BulkShardOperationsTests.java index e3f26eed0c2e9..800fd27c4f5f7 100644 --- a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/bulk/BulkShardOperationsTests.java +++ b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/bulk/BulkShardOperationsTests.java @@ -62,7 +62,7 @@ public void testPrimaryTermFromFollower() throws IOException { () -> randomFrom(Translog.Operation.Type.values()) ); switch (type) { - case INDEX -> operations.add(new Translog.Index(id, seqNo, primaryTerm, 0, SOURCE, null, -1)); + case INDEX -> operations.add(new Translog.Index(id, seqNo, primaryTerm, 0, SOURCE, null, -1, null)); case DELETE -> operations.add(new Translog.Delete(id, seqNo, primaryTerm, 0)); case NO_OP -> operations.add(new Translog.NoOp(seqNo, primaryTerm, "test")); default -> throw new IllegalStateException("unexpected operation type [" + type + "]"); @@ -122,7 +122,7 @@ public void testPrimaryResultIncludeOnlyAppliedOperations() throws Exception { final String id = Integer.toString(between(1, 100)); final Translog.Operation op; if (randomBoolean()) { - op = new Translog.Index(id, seqno++, primaryTerm, 0, SOURCE, null, -1); + op = new Translog.Index(id, seqno++, primaryTerm, 0, SOURCE, null, -1, null); } else if (randomBoolean()) { op = new Translog.Delete(id, seqno++, primaryTerm, 0); } else { diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/SemanticInferenceMetadataFieldsRecoveryTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/SemanticInferenceMetadataFieldsRecoveryTests.java index 175c3e90f798d..e84b5993cbfa0 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/SemanticInferenceMetadataFieldsRecoveryTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/SemanticInferenceMetadataFieldsRecoveryTests.java @@ -168,7 +168,8 @@ public void testSnapshotRecovery() throws IOException { result.getVersion(), op.source(), op.routing(), - op.getAutoGeneratedIdTimestamp() + op.getAutoGeneratedIdTimestamp(), + op.tsid() ) );