diff --git a/server/src/main/java/org/elasticsearch/action/index/IndexRequest.java b/server/src/main/java/org/elasticsearch/action/index/IndexRequest.java index 0e6917d1f5ac4..e3fe113c2b976 100644 --- a/server/src/main/java/org/elasticsearch/action/index/IndexRequest.java +++ b/server/src/main/java/org/elasticsearch/action/index/IndexRequest.java @@ -923,7 +923,7 @@ public Index getConcreteWriteIndex(IndexAbstraction ia, ProjectMetadata project) @Override public int route(IndexRouting indexRouting) { - return indexRouting.indexShard(id, routing, tsid, indexSource.contentType(), indexSource.bytes()); + return indexRouting.indexShard(this); } public IndexRequest setRequireAlias(boolean requireAlias) { diff --git a/server/src/main/java/org/elasticsearch/cluster/routing/IndexRouting.java b/server/src/main/java/org/elasticsearch/cluster/routing/IndexRouting.java index 304383d13e4aa..b7d69cc9ff54c 100644 --- a/server/src/main/java/org/elasticsearch/cluster/routing/IndexRouting.java +++ b/server/src/main/java/org/elasticsearch/cluster/routing/IndexRouting.java @@ -19,7 +19,6 @@ import org.elasticsearch.cluster.metadata.IndexReshardingState; import org.elasticsearch.cluster.metadata.MappingMetadata; import org.elasticsearch.common.ParsingException; -import org.elasticsearch.common.Strings; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.regex.Regex; import org.elasticsearch.common.util.ByteUtils; @@ -32,24 +31,18 @@ import org.elasticsearch.index.mapper.TimeSeriesRoutingHashFieldMapper; import org.elasticsearch.transport.Transports; import org.elasticsearch.xcontent.XContentParser; -import org.elasticsearch.xcontent.XContentParser.Token; import org.elasticsearch.xcontent.XContentParserConfiguration; -import org.elasticsearch.xcontent.XContentString; import org.elasticsearch.xcontent.XContentType; import java.io.IOException; -import java.util.ArrayList; import java.util.Base64; -import java.util.Collections; import java.util.List; import java.util.OptionalInt; import java.util.Set; import java.util.function.IntConsumer; -import java.util.function.IntSupplier; import java.util.function.Predicate; import static org.elasticsearch.common.xcontent.XContentParserUtils.ensureExpectedToken; -import static org.elasticsearch.common.xcontent.XContentParserUtils.expectValueToken; /** * Generates the shard id for {@code (id, routing)} pairs. @@ -62,8 +55,13 @@ public abstract class IndexRouting { * Build the routing from {@link IndexMetadata}. */ public static IndexRouting fromIndexMetadata(IndexMetadata metadata) { - if (metadata.getRoutingPaths().isEmpty() == false || metadata.getTimeSeriesDimensions().isEmpty() == false) { - return new ExtractFromSource(metadata); + if (metadata.getIndexMode() == IndexMode.TIME_SERIES + && metadata.getTimeSeriesDimensions().isEmpty() == false + && metadata.getCreationVersion().onOrAfter(IndexVersions.TSID_CREATED_DURING_ROUTING)) { + return new ExtractFromSource.ForIndexDimensions(metadata); + } + if (metadata.getRoutingPaths().isEmpty() == false) { + return new ExtractFromSource.ForRoutingPath(metadata); } if (metadata.isRoutingPartitionedIndex()) { return new Partitioned(metadata); @@ -97,13 +95,7 @@ public void postProcess(IndexRequest indexRequest) {} * Called when indexing a document to generate the shard id that should contain * a document with the provided parameters. */ - public abstract int indexShard( - String id, - @Nullable String routing, - @Nullable BytesRef tsid, - XContentType sourceType, - BytesReference source - ); + public abstract int indexShard(IndexRequest indexRequest); /** * Called when updating a document to generate the shard id that should contain @@ -223,13 +215,9 @@ private static boolean isNewIndexVersion(final IndexVersion creationVersion) { } @Override - public int indexShard( - String id, - @Nullable String routing, - @Nullable BytesRef tsid, - XContentType sourceType, - BytesReference source - ) { + public int indexShard(IndexRequest indexRequest) { + String id = indexRequest.id(); + String routing = indexRequest.routing(); if (id == null) { throw new IllegalStateException("id is required and should have been set by process"); } @@ -313,60 +301,31 @@ public void collectSearchShards(String routing, IntConsumer consumer) { } } - public static class ExtractFromSource extends IndexRouting { - private final Predicate isRoutingPath; - private final XContentParserConfiguration parserConfig; + /** + * Base class for strategies that determine the shard by extracting and hashing fields from the document source. + */ + public abstract static class ExtractFromSource extends IndexRouting { + protected final XContentParserConfiguration parserConfig; private final IndexMode indexMode; private final boolean trackTimeSeriesRoutingHash; - private final boolean createTsidDuringRouting; private final boolean addIdWithRoutingHash; private int hash = Integer.MAX_VALUE; - @Nullable - private BytesRef tsid; - ExtractFromSource(IndexMetadata metadata) { + ExtractFromSource(IndexMetadata metadata, List includePaths) { super(metadata); if (metadata.isRoutingPartitionedIndex()) { throw new IllegalArgumentException("routing_partition_size is incompatible with routing_path"); } indexMode = metadata.getIndexMode(); assert indexMode != null : "Index mode must be set for ExtractFromSource routing"; - var createTsidDuringRouting = false; - var trackTimeSeriesRoutingHash = false; - List includePaths; - includePaths = metadata.getRoutingPaths(); - if (indexMode == IndexMode.TIME_SERIES) { - if (metadata.getTimeSeriesDimensions().isEmpty() == false - && metadata.getCreationVersion().onOrAfter(IndexVersions.TSID_CREATED_DURING_ROUTING)) { - // This optimization is only available for new indices where - // the dimensions index setting is automatically populated from the mappings. - // If users manually set the routing paths, the optimization is not applied. - createTsidDuringRouting = true; - includePaths = metadata.getTimeSeriesDimensions(); - } - if (metadata.getCreationVersion().onOrAfter(IndexVersions.TIME_SERIES_ROUTING_HASH_IN_ID)) { - trackTimeSeriesRoutingHash = true; - } - } - this.createTsidDuringRouting = createTsidDuringRouting; - this.trackTimeSeriesRoutingHash = trackTimeSeriesRoutingHash; + this.trackTimeSeriesRoutingHash = indexMode == IndexMode.TIME_SERIES + && metadata.getCreationVersion().onOrAfter(IndexVersions.TIME_SERIES_ROUTING_HASH_IN_ID); addIdWithRoutingHash = indexMode == IndexMode.LOGSDB; - isRoutingPath = Regex.simpleMatcher(includePaths.toArray(String[]::new)); this.parserConfig = XContentParserConfiguration.EMPTY.withFiltering(null, Set.copyOf(includePaths), null, true); } - public boolean matchesField(String fieldName) { - return isRoutingPath.test(fieldName); - } - @Override public void postProcess(IndexRequest indexRequest) { - // Update the request with the routing hash and the tsid, if needed. - // This needs to happen in post-processing, after the routing hash is calculated. - if (createTsidDuringRouting) { - assert tsid != null; - indexRequest.tsid(tsid); - } if (trackTimeSeriesRoutingHash) { indexRequest.routing(TimeSeriesRoutingHashFieldMapper.encode(hash)); } else if (addIdWithRoutingHash) { @@ -376,171 +335,21 @@ public void postProcess(IndexRequest indexRequest) { } @Override - public int indexShard( - String id, - @Nullable String routing, - @Nullable BytesRef tsid, - XContentType sourceType, - BytesReference source - ) { + public int indexShard(IndexRequest indexRequest) { assert Transports.assertNotTransportThread("parsing the _source can get slow"); - checkNoRouting(routing); - if (createTsidDuringRouting) { - if (tsid == null) { - this.tsid = buildTsid(sourceType, source); - } else { - this.tsid = tsid; - } - hash = hash(this.tsid); - } else { - hash = hashRoutingFields(sourceType, source).buildHash(IndexRouting.ExtractFromSource::defaultOnEmpty); - } + checkNoRouting(indexRequest.routing()); + hash = hashSource(indexRequest); int shardId = hashToShardId(hash); return (rerouteWritesIfResharding(shardId)); } - public String createId(XContentType sourceType, BytesReference source, byte[] suffix) { - return hashRoutingFields(sourceType, source).createId(suffix, IndexRouting.ExtractFromSource::defaultOnEmpty); - } + protected abstract int hashSource(IndexRequest indexRequest); private static int defaultOnEmpty() { throw new IllegalArgumentException("Error extracting routing: source didn't contain any routing fields"); } - public RoutingHashBuilder builder() { - return new RoutingHashBuilder(); - } - - private RoutingHashBuilder hashRoutingFields(XContentType sourceType, BytesReference source) { - RoutingHashBuilder b = builder(); - try (XContentParser parser = XContentHelper.createParserNotCompressed(parserConfig, source, sourceType)) { - parser.nextToken(); // Move to first token - if (parser.currentToken() == null) { - throw new IllegalArgumentException("Error extracting routing: source didn't contain any routing fields"); - } - parser.nextToken(); - b.extractObject(null, parser); - ensureExpectedToken(null, parser.nextToken(), parser); - } catch (IOException | ParsingException e) { - throw new IllegalArgumentException("Error extracting routing: " + e.getMessage(), e); - } - return b; - } - - private BytesRef buildTsid(XContentType sourceType, BytesReference source) { - TsidBuilder b = new TsidBuilder(); - try (XContentParser parser = XContentHelper.createParserNotCompressed(parserConfig, source, sourceType)) { - b.add(parser, XContentParserTsidFunnel.get()); - } catch (IOException | ParsingException e) { - throw new IllegalArgumentException("Error extracting tsid: " + e.getMessage(), e); - } - return b.buildTsid(); - } - - /** - * Visible for testing - */ - boolean isCreateTsidDuringRouting() { - return createTsidDuringRouting; - } - - public class RoutingHashBuilder { - private final List hashes = new ArrayList<>(); - - public void addMatching(String fieldName, BytesRef string) { - if (isRoutingPath.test(fieldName)) { - addHash(fieldName, string); - } - } - - /** - * Only expected to be called for old indices created before - * {@link IndexVersions#TIME_SERIES_ROUTING_HASH_IN_ID} while creating (during ingestion) - * or synthesizing (at query time) the _id field. - */ - public String createId(byte[] suffix, IntSupplier onEmpty) { - byte[] idBytes = new byte[4 + suffix.length]; - ByteUtils.writeIntLE(buildHash(onEmpty), idBytes, 0); - System.arraycopy(suffix, 0, idBytes, 4, suffix.length); - return Strings.BASE_64_NO_PADDING_URL_ENCODER.encodeToString(idBytes); - } - - private void extractObject(@Nullable String path, XContentParser source) throws IOException { - while (source.currentToken() != Token.END_OBJECT) { - ensureExpectedToken(Token.FIELD_NAME, source.currentToken(), source); - String fieldName = source.currentName(); - String subPath = path == null ? fieldName : path + "." + fieldName; - source.nextToken(); - extractItem(subPath, source); - } - } - - private void extractArray(@Nullable String path, XContentParser source) throws IOException { - while (source.currentToken() != Token.END_ARRAY) { - expectValueToken(source.currentToken(), source); - extractItem(path, source); - } - } - - private void extractItem(String path, XContentParser source) throws IOException { - switch (source.currentToken()) { - case START_OBJECT: - source.nextToken(); - extractObject(path, source); - source.nextToken(); - break; - case VALUE_STRING: - case VALUE_NUMBER: - case VALUE_BOOLEAN: - XContentString.UTF8Bytes utf8Bytes = source.optimizedText().bytes(); - addHash(path, new BytesRef(utf8Bytes.bytes(), utf8Bytes.offset(), utf8Bytes.length())); - source.nextToken(); - break; - case START_ARRAY: - source.nextToken(); - extractArray(path, source); - source.nextToken(); - break; - case VALUE_NULL: - source.nextToken(); - break; - default: - throw new ParsingException( - source.getTokenLocation(), - "Cannot extract routing path due to unexpected token [{}]", - source.currentToken() - ); - } - } - - private void addHash(String path, BytesRef value) { - hashes.add(new NameAndHash(new BytesRef(path), hash(value), hashes.size())); - } - - private int buildHash(IntSupplier onEmpty) { - if (hashes.isEmpty()) { - return onEmpty.getAsInt(); - } - Collections.sort(hashes); - int hash = 0; - for (NameAndHash nah : hashes) { - hash = 31 * hash + (hash(nah.name) ^ nah.hash); - } - return hash; - } - - private record NameAndHash(BytesRef name, int hash, int order) implements Comparable { - @Override - public int compareTo(NameAndHash o) { - int i = name.compareTo(o.name); - if (i != 0) return i; - // ensures array values are in the order as they appear in the source - return Integer.compare(order, o.order); - } - } - } - - private static int hash(BytesRef ref) { + protected static int hash(BytesRef ref) { return StringHelper.murmurhash3_x86_32(ref, 0); } @@ -598,5 +407,97 @@ public void collectSearchShards(String routing, IntConsumer consumer) { private String error(String operation) { return operation + " is not supported because the destination index [" + indexName + "] is in " + indexMode.getName() + " mode"; } + + /** + * Strategy for indices that use {@link IndexMetadata#INDEX_ROUTING_PATH} to extract the routing value from the source. + * This is used primarily for time-series indices created before {@link IndexVersions#TSID_CREATED_DURING_ROUTING} + * and for LogsDB indices that route on specific fields. + * For time-series indices this strategy will result in dimensions to be extracted and hashed twice during indexing: + * once in the coordinating node during shard routing and then again in the data node to create the tsid during document parsing. + * The {@link ForIndexDimensions} strategy avoids this double hashing. + */ + public static class ForRoutingPath extends ExtractFromSource { + private final Predicate isRoutingPath; + + ForRoutingPath(IndexMetadata metadata) { + super(metadata, metadata.getRoutingPaths()); + isRoutingPath = Regex.simpleMatcher(metadata.getRoutingPaths().toArray(String[]::new)); + } + + @Override + protected int hashSource(IndexRequest indexRequest) { + return hashRoutingFields(indexRequest.getContentType(), indexRequest.source()).buildHash( + IndexRouting.ExtractFromSource::defaultOnEmpty + ); + } + + public String createId(XContentType sourceType, BytesReference source, byte[] suffix) { + return hashRoutingFields(sourceType, source).createId(suffix, IndexRouting.ExtractFromSource::defaultOnEmpty); + } + + public RoutingHashBuilder builder() { + return new RoutingHashBuilder(isRoutingPath); + } + + private RoutingHashBuilder hashRoutingFields(XContentType sourceType, BytesReference source) { + RoutingHashBuilder b = builder(); + try (XContentParser parser = XContentHelper.createParserNotCompressed(parserConfig, source, sourceType)) { + parser.nextToken(); // Move to first token + if (parser.currentToken() == null) { + throw new IllegalArgumentException("Error extracting routing: source didn't contain any routing fields"); + } + parser.nextToken(); + b.extractObject(null, parser); + ensureExpectedToken(null, parser.nextToken(), parser); + } catch (IOException | ParsingException e) { + throw new IllegalArgumentException("Error extracting routing: " + e.getMessage(), e); + } + return b; + } + + public boolean matchesField(String fieldName) { + return isRoutingPath.test(fieldName); + } + } + + /** + * Strategy for time-series indices that use {@link IndexMetadata#INDEX_DIMENSIONS} to extract the tsid from the source. + * This strategy avoids double hashing of dimensions during indexing. + * It requires that the index was created with {@link IndexVersions#TSID_CREATED_DURING_ROUTING} or later. + * It creates the tsid during routing and makes the routing decision based on the tsid. + * The tsid gets attached to the index request so that the data node can reuse it instead of rebuilding it. + */ + public static class ForIndexDimensions extends ExtractFromSource { + + ForIndexDimensions(IndexMetadata metadata) { + super(metadata, metadata.getTimeSeriesDimensions()); + assert metadata.getIndexMode() == IndexMode.TIME_SERIES : "Index mode must be time_series for ForIndexDimensions routing"; + assert metadata.getCreationVersion().onOrAfter(IndexVersions.TSID_CREATED_DURING_ROUTING) + : "Index version must be at least " + + IndexVersions.TSID_CREATED_DURING_ROUTING + + " for ForIndexDimensions routing but was " + + metadata.getCreationVersion(); + } + + @Override + protected int hashSource(IndexRequest indexRequest) { + BytesRef tsid = indexRequest.tsid(); + if (tsid == null) { + tsid = buildTsid(indexRequest.getContentType(), indexRequest.indexSource().bytes()); + indexRequest.tsid(tsid); + } + return hash(tsid); + } + + public BytesRef buildTsid(XContentType sourceType, BytesReference source) { + TsidBuilder b = new TsidBuilder(); + try (XContentParser parser = XContentHelper.createParserNotCompressed(parserConfig, source, sourceType)) { + b.add(parser, XContentParserTsidFunnel.get()); + } catch (IOException | ParsingException e) { + throw new IllegalArgumentException("Error extracting tsid: " + e.getMessage(), e); + } + return b.buildTsid(); + } + } } } diff --git a/server/src/main/java/org/elasticsearch/cluster/routing/RoutingHashBuilder.java b/server/src/main/java/org/elasticsearch/cluster/routing/RoutingHashBuilder.java new file mode 100644 index 0000000000000..c542707eb2d84 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/cluster/routing/RoutingHashBuilder.java @@ -0,0 +1,135 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.cluster.routing; + +import org.apache.lucene.util.BytesRef; +import org.elasticsearch.common.ParsingException; +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.util.ByteUtils; +import org.elasticsearch.core.Nullable; +import org.elasticsearch.index.IndexVersions; +import org.elasticsearch.xcontent.XContentParser; +import org.elasticsearch.xcontent.XContentString; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.function.IntSupplier; +import java.util.function.Predicate; + +import static org.elasticsearch.common.xcontent.XContentParserUtils.ensureExpectedToken; +import static org.elasticsearch.common.xcontent.XContentParserUtils.expectValueToken; + +/** + * A builder for computing a hash from fields in the document source that are part of the + * {@link org.elasticsearch.cluster.metadata.IndexMetadata#INDEX_ROUTING_PATH}. + * It is used in the context of {@link IndexRouting.ExtractFromSource.ForRoutingPath} to determine the shard a document should be routed to. + */ +public class RoutingHashBuilder { + private final List hashes = new ArrayList<>(); + private final Predicate isRoutingPath; + + public RoutingHashBuilder(Predicate isRoutingPath) { + this.isRoutingPath = isRoutingPath; + } + + public void addMatching(String fieldName, BytesRef string) { + if (isRoutingPath.test(fieldName)) { + addHash(fieldName, string); + } + } + + /** + * Only expected to be called for old indices created before + * {@link IndexVersions#TIME_SERIES_ROUTING_HASH_IN_ID} while creating (during ingestion) + * or synthesizing (at query time) the _id field. + */ + public String createId(byte[] suffix, IntSupplier onEmpty) { + byte[] idBytes = new byte[4 + suffix.length]; + ByteUtils.writeIntLE(buildHash(onEmpty), idBytes, 0); + System.arraycopy(suffix, 0, idBytes, 4, suffix.length); + return Strings.BASE_64_NO_PADDING_URL_ENCODER.encodeToString(idBytes); + } + + void extractObject(@Nullable String path, XContentParser source) throws IOException { + while (source.currentToken() != XContentParser.Token.END_OBJECT) { + ensureExpectedToken(XContentParser.Token.FIELD_NAME, source.currentToken(), source); + String fieldName = source.currentName(); + String subPath = path == null ? fieldName : path + "." + fieldName; + source.nextToken(); + extractItem(subPath, source); + } + } + + private void extractArray(@Nullable String path, XContentParser source) throws IOException { + while (source.currentToken() != XContentParser.Token.END_ARRAY) { + expectValueToken(source.currentToken(), source); + extractItem(path, source); + } + } + + private void extractItem(String path, XContentParser source) throws IOException { + switch (source.currentToken()) { + case START_OBJECT: + source.nextToken(); + extractObject(path, source); + source.nextToken(); + break; + case VALUE_STRING: + case VALUE_NUMBER: + case VALUE_BOOLEAN: + XContentString.UTF8Bytes utf8Bytes = source.optimizedText().bytes(); + addHash(path, new BytesRef(utf8Bytes.bytes(), utf8Bytes.offset(), utf8Bytes.length())); + source.nextToken(); + break; + case START_ARRAY: + source.nextToken(); + extractArray(path, source); + source.nextToken(); + break; + case VALUE_NULL: + source.nextToken(); + break; + default: + throw new ParsingException( + source.getTokenLocation(), + "Cannot extract routing path due to unexpected token [{}]", + source.currentToken() + ); + } + } + + private void addHash(String path, BytesRef value) { + hashes.add(new NameAndHash(new BytesRef(path), IndexRouting.ExtractFromSource.hash(value), hashes.size())); + } + + int buildHash(IntSupplier onEmpty) { + if (hashes.isEmpty()) { + return onEmpty.getAsInt(); + } + Collections.sort(hashes); + int hash = 0; + for (NameAndHash nah : hashes) { + hash = 31 * hash + (IndexRouting.ExtractFromSource.hash(nah.name) ^ nah.hash); + } + return hash; + } + + private record NameAndHash(BytesRef name, int hash, int order) implements Comparable { + @Override + public int compareTo(NameAndHash o) { + int i = name.compareTo(o.name); + if (i != 0) return i; + // ensures array values are in the order as they appear in the source + return Integer.compare(order, o.order); + } + } +} diff --git a/server/src/main/java/org/elasticsearch/index/IndexMode.java b/server/src/main/java/org/elasticsearch/index/IndexMode.java index 687d48ef1cc25..46c790a4b5b70 100644 --- a/server/src/main/java/org/elasticsearch/index/IndexMode.java +++ b/server/src/main/java/org/elasticsearch/index/IndexMode.java @@ -35,7 +35,6 @@ import org.elasticsearch.index.mapper.RoutingFields; import org.elasticsearch.index.mapper.RoutingPathFields; import org.elasticsearch.index.mapper.SourceFieldMapper; -import org.elasticsearch.index.mapper.SourceToParse; import org.elasticsearch.index.mapper.TimeSeriesIdFieldMapper; import org.elasticsearch.index.mapper.TimeSeriesRoutingHashFieldMapper; import org.elasticsearch.index.mapper.TsidExtractingIdFieldMapper; @@ -113,7 +112,7 @@ public IdFieldMapper buildIdFieldMapper(BooleanSupplier fieldDataEnabled) { } @Override - public RoutingFields buildRoutingFields(IndexSettings settings, SourceToParse source) { + public RoutingFields buildRoutingFields(IndexSettings settings) { return RoutingFields.Noop.INSTANCE; } @@ -217,13 +216,15 @@ public IdFieldMapper buildIdFieldMapper(BooleanSupplier fieldDataEnabled) { } @Override - public RoutingFields buildRoutingFields(IndexSettings settings, SourceToParse source) { - if (source.tsid() != null) { - // If the source already has a _tsid field, we don't need to extract routing from the source. + public RoutingFields buildRoutingFields(IndexSettings settings) { + IndexRouting indexRouting = settings.getIndexRouting(); + if (indexRouting instanceof IndexRouting.ExtractFromSource.ForRoutingPath forRoutingPath) { + return new RoutingPathFields(forRoutingPath.builder()); + } else if (indexRouting instanceof IndexRouting.ExtractFromSource.ForIndexDimensions) { return RoutingFields.Noop.INSTANCE; + } else { + throw new IllegalStateException("Index routing strategy not supported for index_mode=time_series: " + indexRouting); } - IndexRouting.ExtractFromSource routing = (IndexRouting.ExtractFromSource) settings.getIndexRouting(); - return new RoutingPathFields(routing.builder()); } @Override @@ -302,7 +303,7 @@ public MetadataFieldMapper timeSeriesRoutingHashFieldMapper() { } @Override - public RoutingFields buildRoutingFields(IndexSettings settings, SourceToParse source) { + public RoutingFields buildRoutingFields(IndexSettings settings) { return RoutingFields.Noop.INSTANCE; } @@ -383,7 +384,7 @@ public IdFieldMapper buildIdFieldMapper(BooleanSupplier fieldDataEnabled) { } @Override - public RoutingFields buildRoutingFields(IndexSettings settings, SourceToParse source) { + public RoutingFields buildRoutingFields(IndexSettings settings) { return RoutingFields.Noop.INSTANCE; } @@ -539,7 +540,7 @@ public String getName() { /** * How {@code time_series_dimension} fields are handled by indices in this mode. */ - public abstract RoutingFields buildRoutingFields(IndexSettings settings, SourceToParse source); + public abstract RoutingFields buildRoutingFields(IndexSettings settings); /** * @return Whether timestamps should be validated for being withing the time range of an index. diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java b/server/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java index edf60c69460f6..4112f9108d3ee 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java @@ -13,6 +13,7 @@ import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.search.Query; import org.apache.lucene.util.BytesRef; +import org.elasticsearch.cluster.routing.IndexRouting; import org.elasticsearch.common.Explicit; import org.elasticsearch.common.regex.Regex; import org.elasticsearch.common.xcontent.XContentHelper; @@ -96,7 +97,7 @@ public ParsedDocument parseDocument(SourceToParse source, MappingLookup mappingL ) ) ) { - context = new RootDocumentParserContext(mappingLookup, mappingParserContext, source, parser, source.tsid()); + context = new RootDocumentParserContext(mappingLookup, mappingParserContext, source, parser); validateStart(context.parser()); MetadataFieldMapper[] metadataFieldsMappers = mappingLookup.getMapping().getSortedMetadataMappers(); internalParseDocument(metadataFieldsMappers, context); @@ -1077,8 +1078,7 @@ private static class RootDocumentParserContext extends DocumentParserContext { MappingLookup mappingLookup, MappingParserContext mappingParserContext, SourceToParse source, - XContentParser parser, - BytesRef tsid + XContentParser parser ) throws IOException { super( mappingLookup, @@ -1087,8 +1087,17 @@ private static class RootDocumentParserContext extends DocumentParserContext { mappingLookup.getMapping().getRoot(), ObjectMapper.Dynamic.getRootDynamic(mappingLookup) ); + IndexSettings indexSettings = mappingParserContext.getIndexSettings(); + BytesRef tsid = source.tsid(); + if (tsid == null + && indexSettings.getMode() == IndexMode.TIME_SERIES + && indexSettings.getIndexRouting() instanceof IndexRouting.ExtractFromSource.ForIndexDimensions forIndexDimensions) { + // the tsid is normally set on the coordinating node during shard routing and passed to the data node via the index request + // but when applying a translog operation, shard routing is not happening, and we have to create the tsid from source + tsid = forIndexDimensions.buildTsid(source.getXContentType(), source.source()); + } this.tsid = tsid; - assert tsid == null || mappingParserContext.getIndexSettings().getMode() == IndexMode.TIME_SERIES + assert this.tsid == null || indexSettings.getMode() == IndexMode.TIME_SERIES : "tsid should only be set for time series indices"; if (mappingLookup.getMapping().getRoot().subobjects() == ObjectMapper.Subobjects.ENABLED) { this.parser = DotExpandingXContentParser.expandDots(parser, this.path); diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DocumentParserContext.java b/server/src/main/java/org/elasticsearch/index/mapper/DocumentParserContext.java index 28cd1c4533db0..0b1a64713857a 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DocumentParserContext.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DocumentParserContext.java @@ -271,7 +271,7 @@ protected DocumentParserContext( null, null, SeqNoFieldMapper.SequenceIDFields.emptySeqID(mappingParserContext.getIndexSettings().seqNoIndexOptions()), - RoutingFields.fromIndexSettings(mappingParserContext.getIndexSettings(), source), + RoutingFields.fromIndexSettings(mappingParserContext.getIndexSettings()), parent, dynamic, new HashSet<>(), diff --git a/server/src/main/java/org/elasticsearch/index/mapper/IdLoader.java b/server/src/main/java/org/elasticsearch/index/mapper/IdLoader.java index 12228760faf35..9ceae1c750733 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/IdLoader.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/IdLoader.java @@ -17,6 +17,7 @@ import org.apache.lucene.util.BytesRef; import org.elasticsearch.cluster.metadata.DataStream; import org.elasticsearch.cluster.routing.IndexRouting; +import org.elasticsearch.cluster.routing.RoutingHashBuilder; import org.elasticsearch.index.fieldvisitor.LeafStoredFieldLoader; import java.io.IOException; @@ -37,7 +38,7 @@ static IdLoader fromLeafStoredFieldLoader() { /** * @return returns an {@link IdLoader} instance that syn synthesizes _id from routing, _tsid and @timestamp fields. */ - static IdLoader createTsIdLoader(IndexRouting.ExtractFromSource indexRouting, List routingPaths) { + static IdLoader createTsIdLoader(IndexRouting.ExtractFromSource.ForRoutingPath indexRouting, List routingPaths) { return new TsIdLoader(indexRouting, routingPaths); } @@ -58,19 +59,19 @@ sealed interface Leaf permits StoredLeaf, TsIdLeaf { final class TsIdLoader implements IdLoader { - private final IndexRouting.ExtractFromSource indexRouting; + private final IndexRouting.ExtractFromSource.ForRoutingPath indexRouting; private final List routingPaths; - TsIdLoader(IndexRouting.ExtractFromSource indexRouting, List routingPaths) { + TsIdLoader(IndexRouting.ExtractFromSource.ForRoutingPath indexRouting, List routingPaths) { this.routingPaths = routingPaths; this.indexRouting = indexRouting; } public IdLoader.Leaf leaf(LeafStoredFieldLoader loader, LeafReader reader, int[] docIdsInLeaf) throws IOException { - IndexRouting.ExtractFromSource.RoutingHashBuilder[] builders = null; + RoutingHashBuilder[] builders = null; if (indexRouting != null) { // this branch is for legacy indices before IndexVersions.TIME_SERIES_ROUTING_HASH_IN_ID - builders = new IndexRouting.ExtractFromSource.RoutingHashBuilder[docIdsInLeaf.length]; + builders = new RoutingHashBuilder[docIdsInLeaf.length]; for (int i = 0; i < builders.length; i++) { builders[i] = indexRouting.builder(); } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/RoutingFields.java b/server/src/main/java/org/elasticsearch/index/mapper/RoutingFields.java index 0ad4f4e0b49aa..304a57bc647a4 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/RoutingFields.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/RoutingFields.java @@ -22,8 +22,8 @@ public interface RoutingFields { /** * Collect routing fields from index settings */ - static RoutingFields fromIndexSettings(IndexSettings indexSettings, SourceToParse source) { - return indexSettings.getMode().buildRoutingFields(indexSettings, source); + static RoutingFields fromIndexSettings(IndexSettings indexSettings) { + return indexSettings.getMode().buildRoutingFields(indexSettings); } /** diff --git a/server/src/main/java/org/elasticsearch/index/mapper/RoutingPathFields.java b/server/src/main/java/org/elasticsearch/index/mapper/RoutingPathFields.java index 361163ea1be66..d30b307843a84 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/RoutingPathFields.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/RoutingPathFields.java @@ -11,7 +11,7 @@ import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.StringHelper; -import org.elasticsearch.cluster.routing.IndexRouting; +import org.elasticsearch.cluster.routing.RoutingHashBuilder; import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.hash.Murmur3Hasher; @@ -59,9 +59,9 @@ public final class RoutingPathFields implements RoutingFields { * Builds the routing. Used for building {@code _id}. If null then skipped. */ @Nullable - private final IndexRouting.ExtractFromSource.RoutingHashBuilder routingBuilder; + private final RoutingHashBuilder routingBuilder; - public RoutingPathFields(@Nullable IndexRouting.ExtractFromSource.RoutingHashBuilder routingBuilder) { + public RoutingPathFields(@Nullable RoutingHashBuilder routingBuilder) { this.routingBuilder = routingBuilder; } @@ -69,7 +69,7 @@ SortedMap> routingValues() { return Collections.unmodifiableSortedMap(routingValues); } - IndexRouting.ExtractFromSource.RoutingHashBuilder routingBuilder() { + RoutingHashBuilder routingBuilder() { return routingBuilder; } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapper.java index 0f3776f1db9bb..288ddc116a08e 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapper.java @@ -14,7 +14,7 @@ import org.apache.lucene.document.StringField; import org.apache.lucene.search.Query; import org.apache.lucene.util.BytesRef; -import org.elasticsearch.cluster.routing.IndexRouting; +import org.elasticsearch.cluster.routing.RoutingHashBuilder; import org.elasticsearch.common.Strings; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.io.stream.BytesStreamOutput; @@ -161,12 +161,13 @@ public void postParse(DocumentParserContext context) throws IOException { throw new MapperException("Too many dimension fields [" + size + "], max [" + limit + "] dimension fields allowed"); } timeSeriesId = buildLegacyTsid(routingPathFields).toBytesRef(); - } else if (context.getTsid() != null) { + } else if (context.getRoutingFields() instanceof RoutingPathFields routingPathFieldsFromContext) { + routingPathFields = routingPathFieldsFromContext; + timeSeriesId = routingPathFields.buildHash().toBytesRef(); + } else { routingPathFields = null; + assert context.getTsid() != null; timeSeriesId = context.getTsid(); - } else { - routingPathFields = (RoutingPathFields) context.getRoutingFields(); - timeSeriesId = routingPathFields.buildHash().toBytesRef(); } if (this.useDocValuesSkipper) { @@ -175,7 +176,7 @@ public void postParse(DocumentParserContext context) throws IOException { context.doc().add(new SortedDocValuesField(fieldType().name(), timeSeriesId)); } - IndexRouting.ExtractFromSource.RoutingHashBuilder routingBuilder; + RoutingHashBuilder routingBuilder; if (getIndexVersionCreated(context).before(IndexVersions.TIME_SERIES_ROUTING_HASH_IN_ID) && routingPathFields != null) { // For legacy indices, we need to create the routing hash from the routing path fields. routingBuilder = routingPathFields.routingBuilder(); diff --git a/server/src/main/java/org/elasticsearch/index/mapper/TsidExtractingIdFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/TsidExtractingIdFieldMapper.java index b48b22520ea36..bb8b0d9ec775c 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/TsidExtractingIdFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/TsidExtractingIdFieldMapper.java @@ -14,6 +14,7 @@ import org.apache.lucene.index.IndexableField; import org.apache.lucene.util.BytesRef; import org.elasticsearch.cluster.routing.IndexRouting; +import org.elasticsearch.cluster.routing.RoutingHashBuilder; import org.elasticsearch.common.Strings; import org.elasticsearch.common.hash.MurmurHash3; import org.elasticsearch.common.hash.MurmurHash3.Hash128; @@ -46,11 +47,7 @@ public IndexFieldData.Builder fielddataBuilder(FieldDataContext fieldDataContext private static final long SEED = 0; - public static BytesRef createField( - DocumentParserContext context, - IndexRouting.ExtractFromSource.RoutingHashBuilder routingBuilder, - BytesRef tsid - ) { + public static BytesRef createField(DocumentParserContext context, RoutingHashBuilder routingBuilder, BytesRef tsid) { final long timestamp = DataStreamTimestampFieldMapper.extractTimestampValue(context.doc()); String id; if (routingBuilder != null) { @@ -65,7 +62,7 @@ public static BytesRef createField( * at all we just skip the assertion because we can't be sure * it always must pass. */ - IndexRouting.ExtractFromSource indexRouting = (IndexRouting.ExtractFromSource) context.indexSettings().getIndexRouting(); + var indexRouting = (IndexRouting.ExtractFromSource.ForRoutingPath) context.indexSettings().getIndexRouting(); assert context.getDynamicMappers().isEmpty() == false || context.getDynamicRuntimeFields().isEmpty() == false || id.equals(indexRouting.createId(context.sourceToParse().getXContentType(), context.sourceToParse().source(), suffix)); @@ -115,7 +112,7 @@ public static String createId(int routingHash, BytesRef tsid, long timestamp) { public static String createId( boolean dynamicMappersExists, - IndexRouting.ExtractFromSource.RoutingHashBuilder routingBuilder, + RoutingHashBuilder routingBuilder, BytesRef tsid, long timestamp, byte[] suffix diff --git a/server/src/main/java/org/elasticsearch/search/DefaultSearchContext.java b/server/src/main/java/org/elasticsearch/search/DefaultSearchContext.java index c2b526128a9bc..475d2d1887563 100644 --- a/server/src/main/java/org/elasticsearch/search/DefaultSearchContext.java +++ b/server/src/main/java/org/elasticsearch/search/DefaultSearchContext.java @@ -951,10 +951,10 @@ public SourceLoader newSourceLoader(@Nullable SourceFilter filter) { @Override public IdLoader newIdLoader() { if (indexService.getIndexSettings().getMode() == IndexMode.TIME_SERIES) { - IndexRouting.ExtractFromSource indexRouting = null; + IndexRouting.ExtractFromSource.ForRoutingPath indexRouting = null; List routingPaths = null; if (indexService.getIndexSettings().getIndexVersionCreated().before(IndexVersions.TIME_SERIES_ROUTING_HASH_IN_ID)) { - indexRouting = (IndexRouting.ExtractFromSource) indexService.getIndexSettings().getIndexRouting(); + indexRouting = (IndexRouting.ExtractFromSource.ForRoutingPath) indexService.getIndexSettings().getIndexRouting(); routingPaths = indexService.getMetadata().getRoutingPaths(); for (String routingField : routingPaths) { if (routingField.contains("*")) { diff --git a/server/src/test/java/org/elasticsearch/cluster/routing/IndexRoutingTests.java b/server/src/test/java/org/elasticsearch/cluster/routing/IndexRoutingTests.java index d9258be24398a..4f0ed5fce5aa8 100644 --- a/server/src/test/java/org/elasticsearch/cluster/routing/IndexRoutingTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/routing/IndexRoutingTests.java @@ -464,7 +464,7 @@ public void testRequiredRouting() { */ private int shardIdFromSimple(IndexRouting indexRouting, String id, @Nullable String routing) { return switch (between(0, 3)) { - case 0 -> indexRouting.indexShard(id, routing, null, null, null); + case 0 -> indexRouting.indexShard(new IndexRequest().id(id).routing(routing)); case 1 -> indexRouting.updateShard(id, routing); case 2 -> indexRouting.deleteShard(id, routing); case 3 -> indexRouting.getShard(id, routing); @@ -497,7 +497,7 @@ public void testRoutingPathEmptySource() throws IOException { IndexRouting routing = indexRoutingForPath(between(1, 5), randomAlphaOfLength(5)); Exception e = expectThrows( IllegalArgumentException.class, - () -> routing.indexShard(randomAlphaOfLength(5), null, null, XContentType.JSON, source(Map.of())) + () -> routing.indexShard(new IndexRequest().id(randomAlphaOfLength(5)).source(Map.of())) ); assertThat(e.getMessage(), stringContainsInOrder("Error extracting", "source didn't contain any")); } @@ -506,7 +506,7 @@ public void testRoutingPathMismatchSource() throws IOException { IndexRouting routing = indexRoutingForPath(between(1, 5), "foo"); Exception e = expectThrows( IllegalArgumentException.class, - () -> routing.indexShard(randomAlphaOfLength(5), null, null, XContentType.JSON, source(Map.of("bar", "dog"))) + () -> routing.indexShard(new IndexRequest().id(randomAlphaOfLength(5)).source(Map.of("bar", "dog"))) ); assertThat(e.getMessage(), stringContainsInOrder("Error extracting", "source didn't contain any")); } @@ -527,7 +527,9 @@ public void testRoutingIndexWithRouting() throws IOException { String docRouting = randomAlphaOfLength(5); Exception e = expectThrows( IllegalArgumentException.class, - () -> indexRouting.indexShard(randomAlphaOfLength(5), docRouting, null, XContentType.JSON, source) + () -> indexRouting.indexShard( + new IndexRequest().id(randomAlphaOfLength(5)).routing(docRouting).source(source, XContentType.JSON) + ) ); assertThat( e.getMessage(), @@ -620,7 +622,7 @@ public void testRoutingPathObjectArraysInSource() throws IOException { BytesReference source = source(Map.of("a", List.of("foo", Map.of("foo", "bar")))); Exception e = expectThrows( IllegalArgumentException.class, - () -> routing.indexShard(randomAlphaOfLength(5), null, null, XContentType.JSON, source) + () -> routing.indexShard(new IndexRequest().id(randomAlphaOfLength(5)).source(source, XContentType.JSON)) ); assertThat( e.getMessage(), @@ -704,8 +706,8 @@ public void testRoutingPathLogsdb() throws IOException { // Verify that routing uses the field name and value in the routing path. int expectedShard = expectedShard(routing, List.of("foo", "A"), shards); - BytesReference sourceBytes = source(Map.of("foo", "A", "bar", "B")); - assertEquals(expectedShard, routing.indexShard(null, null, null, XContentType.JSON, sourceBytes)); + req.source(Map.of("foo", "A", "bar", "B")); + assertEquals(expectedShard, routing.indexShard(req)); // Verify that the request id gets updated to contain the routing hash. routing.postProcess(req); @@ -726,7 +728,7 @@ public void testCollectSearchShardsUnpartitionedWithResharding() throws IOExcept var shardToRouting = new HashMap(); do { var routing = randomAlphaOfLength(5); - var shard = initialRouting.indexShard("dummy", routing, null, null, null); + var shard = initialRouting.indexShard(new IndexRequest().id("dummy").routing(routing)); if (shardToRouting.containsKey(shard) == false) { shardToRouting.put(shard, routing); } @@ -806,7 +808,7 @@ public void testCollectSearchShardsPartitionedWithResharding() throws IOExceptio var shardToRouting = new TreeMap(); do { var routing = randomAlphaOfLength(5); - var shard = initialRouting.indexShard("dummy", routing, null, null, null); + var shard = initialRouting.indexShard(new IndexRequest().id("dummy").routing(routing)); if (shardToRouting.containsKey(shard) == false) { shardToRouting.put(shard, routing); } @@ -943,28 +945,31 @@ private void assertIndexShard(IndexRouting routing, Map source, private void assertIndexShard(IndexRouting routing, Map source, int expectedShard) throws IOException { byte[] suffix = randomSuffix(); BytesReference sourceBytes = source(source); - assertThat(routing.indexShard(randomAlphaOfLength(5), null, null, XContentType.JSON, sourceBytes), equalTo(expectedShard)); + IndexRequest indexRequest = new IndexRequest(); + indexRequest.source(sourceBytes, XContentType.JSON); + indexRequest.id(randomAlphaOfLength(5)); + routing.preProcess(indexRequest); + assertThat(routing.indexShard(indexRequest), equalTo(expectedShard)); IndexRouting.ExtractFromSource r = (IndexRouting.ExtractFromSource) routing; - if (r.isCreateTsidDuringRouting()) { + if (r instanceof IndexRouting.ExtractFromSource.ForRoutingPath forRoutingPath) { // The rest of the assertions are only relevant when only the routing hash is created - return; - } - String idFromSource = r.createId(XContentType.JSON, sourceBytes, suffix); - assertThat(shardIdForReadFromSourceExtracting(routing, idFromSource), equalTo(expectedShard)); - Map flattened = flatten(source); - String idFromFlattened = r.createId(XContentType.JSON, sourceBytes, suffix); - assertThat(idFromFlattened, equalTo(idFromSource)); - - IndexRouting.ExtractFromSource.RoutingHashBuilder b = r.builder(); - for (Map.Entry e : flattened.entrySet()) { - if (e.getValue() instanceof List listValue) { - listValue.forEach(v -> b.addMatching(e.getKey(), new BytesRef(v.toString()))); - } else { - b.addMatching(e.getKey(), new BytesRef(e.getValue().toString())); + String idFromSource = forRoutingPath.createId(XContentType.JSON, sourceBytes, suffix); + assertThat(shardIdForReadFromSourceExtracting(routing, idFromSource), equalTo(expectedShard)); + Map flattened = flatten(source); + String idFromFlattened = forRoutingPath.createId(XContentType.JSON, sourceBytes, suffix); + assertThat(idFromFlattened, equalTo(idFromSource)); + + RoutingHashBuilder b = forRoutingPath.builder(); + for (Map.Entry e : flattened.entrySet()) { + if (e.getValue() instanceof List listValue) { + listValue.forEach(v -> b.addMatching(e.getKey(), new BytesRef(v.toString()))); + } else { + b.addMatching(e.getKey(), new BytesRef(e.getValue().toString())); + } } + String idFromBuilder = b.createId(suffix, () -> { throw new AssertionError(); }); + assertThat(idFromBuilder, equalTo(idFromSource)); } - String idFromBuilder = b.createId(suffix, () -> { throw new AssertionError(); }); - assertThat(idFromBuilder, equalTo(idFromSource)); } private byte[] randomSuffix() { @@ -1014,7 +1019,7 @@ private int expectedShard(IndexRouting routing, List keysAndValues, int * Build the hash we expect from the extracter. */ private int hash(IndexRouting routing, List keysAndValues) { - if (routing instanceof IndexRouting.ExtractFromSource extractFromSource && extractFromSource.isCreateTsidDuringRouting()) { + if (routing instanceof IndexRouting.ExtractFromSource.ForIndexDimensions) { return tsidBasedRoutingHash(keysAndValues); } return legacyRoutingHash(keysAndValues); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/RoutingPathFieldsTests.java b/server/src/test/java/org/elasticsearch/index/mapper/RoutingPathFieldsTests.java index 2c2c0d160c904..030bb20d63998 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/RoutingPathFieldsTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/RoutingPathFieldsTests.java @@ -30,7 +30,7 @@ public void testWithBuilder() throws Exception { .build(), Settings.EMPTY ); - IndexRouting.ExtractFromSource routing = (IndexRouting.ExtractFromSource) settings.getIndexRouting(); + IndexRouting.ExtractFromSource.ForRoutingPath routing = (IndexRouting.ExtractFromSource.ForRoutingPath) settings.getIndexRouting(); var routingPathFields = new RoutingPathFields(routing.builder()); BytesReference current, previous; diff --git a/server/src/test/java/org/elasticsearch/index/mapper/TsidExtractingIdFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/TsidExtractingIdFieldMapperTests.java index a4a588930dcae..85bd9950ba99c 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/TsidExtractingIdFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/TsidExtractingIdFieldMapperTests.java @@ -36,22 +36,28 @@ public class TsidExtractingIdFieldMapperTests extends MetadataMapperTestCase { private static class TestCase { private final String name; - private final String expectedId; - private final String expectedTsid; + private final String expectedIdWithRoutingPath; + private final String expectedIdWithIndexDimensions; + private final String expectedTsidWithRoutingPath; + private final String expectedTsidWithIndexDimensions; private final String expectedTimestamp; private final CheckedConsumer source; private final List> equivalentSources = new ArrayList<>(); TestCase( String name, - String expectedId, - String expectedTsid, + String expectedIdWithRoutingPath, + String expectedIdWithIndexDimensions, + String expectedTsidWithRoutingPath, + String expectedTsidWithIndexDimensions, String expectedTimestamp, CheckedConsumer source ) { this.name = name; - this.expectedId = expectedId; - this.expectedTsid = expectedTsid; + this.expectedIdWithRoutingPath = expectedIdWithRoutingPath; + this.expectedIdWithIndexDimensions = expectedIdWithIndexDimensions; + this.expectedTsidWithRoutingPath = expectedTsidWithRoutingPath; + this.expectedTsidWithIndexDimensions = expectedTsidWithIndexDimensions; this.expectedTimestamp = expectedTimestamp; this.source = source; } @@ -86,7 +92,9 @@ public static Iterable params() { new TestCase( "2022-01-01T01:00:00Z", "BwAAAKjcFfi45iV3AAABfhMmioA", + "BwAAAEk383E-IPhiAAABfhMmioA", "JJSLNivCxv3hDTQtWd6qGUwGlT_5e6_NYGOZWULpmMG9IAlZlA", + "0XbnnsE9AoHbGpRryIzXGQ0w", "2022-01-01T01:00:00.000Z", b -> { b.field("@timestamp", "2022-01-01T01:00:00Z"); @@ -98,7 +106,9 @@ public static Iterable params() { new TestCase( "2022-01-01T01:00:01Z", "BwAAAKjcFfi45iV3AAABfhMmjmg", + "BwAAAEk383E-IPhiAAABfhMmjmg", "JJSLNivCxv3hDTQtWd6qGUwGlT_5e6_NYGOZWULpmMG9IAlZlA", + "0XbnnsE9AoHbGpRryIzXGQ0w", "2022-01-01T01:00:01.000Z", b -> { b.field("@timestamp", "2022-01-01T01:00:01Z"); @@ -110,7 +120,9 @@ public static Iterable params() { new TestCase( "1970-01-01T00:00:00Z", "BwAAAKjcFfi45iV3AAAAAAAAAAA", + "BwAAAEk383E-IPhiAAAAAAAAAAA", "JJSLNivCxv3hDTQtWd6qGUwGlT_5e6_NYGOZWULpmMG9IAlZlA", + "0XbnnsE9AoHbGpRryIzXGQ0w", "1970-01-01T00:00:00.000Z", b -> { b.field("@timestamp", "1970-01-01T00:00:00Z"); @@ -122,7 +134,9 @@ public static Iterable params() { new TestCase( "-9998-01-01T00:00:00Z", "BwAAAKjcFfi45iV3__6oggRgGAA", + "BwAAAEk383E-IPhi__6oggRgGAA", "JJSLNivCxv3hDTQtWd6qGUwGlT_5e6_NYGOZWULpmMG9IAlZlA", + "0XbnnsE9AoHbGpRryIzXGQ0w", "-9998-01-01T00:00:00.000Z", b -> { b.field("@timestamp", "-9998-01-01T00:00:00Z"); @@ -134,7 +148,9 @@ public static Iterable params() { new TestCase( "9998-01-01T00:00:00Z", "BwAAAKjcFfi45iV3AADmaSK9hAA", + "BwAAAEk383E-IPhiAADmaSK9hAA", "JJSLNivCxv3hDTQtWd6qGUwGlT_5e6_NYGOZWULpmMG9IAlZlA", + "0XbnnsE9AoHbGpRryIzXGQ0w", "9998-01-01T00:00:00.000Z", b -> { b.field("@timestamp", "9998-01-01T00:00:00Z"); @@ -148,7 +164,9 @@ public static Iterable params() { new TestCase( "r1", "BwAAAKjcFfi45iV3AAABfhMmioA", + "BwAAAEk383E-IPhiAAABfhMmioA", "JJSLNivCxv3hDTQtWd6qGUwGlT_5e6_NYGOZWULpmMG9IAlZlA", + "0XbnnsE9AoHbGpRryIzXGQ0w", "2022-01-01T01:00:00.000Z", b -> { b.field("@timestamp", "2022-01-01T01:00:00Z"); @@ -184,7 +202,9 @@ public static Iterable params() { new TestCase( "r2", "BwAAAB0iuE1-sOQpAAABfhMmioA", + "BwAAAAjKcgBsE3XEAAABfhMmioA", "JNY_frTR9GmCbhXgK4Y8W44GlT_5e6_NYGOZWULpmMG9IAlZlA", + "NHaFg-5cCESqVrBaDs6o9DZ4", "2022-01-01T01:00:00.000Z", b -> { b.field("@timestamp", "2022-01-01T01:00:00Z"); @@ -196,7 +216,9 @@ public static Iterable params() { new TestCase( "o.r3", "BwAAAC1h1gf2J5a8AAABfhMmioA", + "BwAAANZEz_KPYz9jAAABfhMmioA", "JEyfZsJIp3UNyfWG-4SjKFIGlT_5e6_NYGOZWULpmMG9IAlZlA", + "K3bwGazghpbQWy_dEU8UL4Ec", "2022-01-01T01:00:00.000Z", b -> { b.field("@timestamp", "2022-01-01T01:00:00Z"); @@ -213,7 +235,9 @@ public static Iterable params() { new TestCase( "k1=dog", "BwAAACrEiVgZlSsYAAABfhMmioA", + "BwAAACyv9sTdBEWNAAABfhMmioA", "KJQKpjU9U63jhh-eNJ1f8bipyU08BpU_-ZJxnTYtoe9Lsg-QvzL-qOY", + "f8B2zv4qzO0Eq9YLPLUIlzRL8g", "2022-01-01T01:00:00.000Z", b -> { b.field("@timestamp", "2022-01-01T01:00:00Z"); @@ -226,7 +250,9 @@ public static Iterable params() { new TestCase( "k1=pumpkin", "BwAAAG8GX8-0QcFxAAABfhMmioA", + "BwAAABp_KncX55XzAAABfhMmioA", "KJQKpjU9U63jhh-eNJ1f8bibzw1JBpU_-VsHjSz5HC1yy_swPEM1iGo", + "f0F2VfGtg5ltThy1tF7mWC_PGQ", "2022-01-01T01:00:00.000Z", b -> { b.field("@timestamp", "2022-01-01T01:00:00Z"); @@ -239,7 +265,9 @@ public static Iterable params() { new TestCase( "k1=empty string", "BwAAAMna58i6D-Q6AAABfhMmioA", + "BwAAAD-6gB-6kGvDAAABfhMmioA", "KJQKpjU9U63jhh-eNJ1f8bhaCD7uBpU_-SWGG0Uv9tZ1mLO2gi9rC1I", + "f8t2FhZTCAA2SSVLPSknT8Nf_A", "2022-01-01T01:00:00.000Z", b -> { b.field("@timestamp", "2022-01-01T01:00:00Z"); @@ -252,7 +280,9 @@ public static Iterable params() { new TestCase( "k2", "BwAAAFqlzAuv-06kAAABfhMmioA", + "BwAAAGEjL8LoBxUJAAABfhMmioA", "KB9H-tGrL_UzqMcqXcgBtzypyU08BpU_-ZJxnTYtoe9Lsg-QvzL-qOY", + "wMB2LHEQrqbi5fgySxCJAU5NhQ", "2022-01-01T01:00:00.000Z", b -> { b.field("@timestamp", "2022-01-01T01:00:00Z"); @@ -265,7 +295,9 @@ public static Iterable params() { new TestCase( "o.k3", "BwAAAC_VhridAKDUAAABfhMmioA", + "BwAAALNlofvJns7KAAABfhMmioA", "KGXATwN7ISd1_EycFRJ9h6qpyU08BpU_-ZJxnTYtoe9Lsg-QvzL-qOY", + "esB2eX4haJwoe0Gz0Hxpr6mIcQ", "2022-01-01T01:00:00.000Z", b -> { b.field("@timestamp", "2022-01-01T01:00:00Z"); @@ -276,9 +308,11 @@ public static Iterable params() { ); items.add( new TestCase( - "o.r3", + "o.r3,o.k3", "BwAAAEwfL7x__2oPAAABfhMmioA", + "BwAAAOPQAwI_cPXTAAABfhMmioA", "KJaYZVZz8plfkEvvPBpi1EWpyU08BpU_-ZJxnTYtoe9Lsg-QvzL-qOY", + "S8B2MTOcF1wV1_fEBh3Xpv0JJA", "2022-01-01T01:00:00.000Z", b -> { b.field("@timestamp", "2022-01-01T01:00:00Z"); @@ -309,7 +343,9 @@ public static Iterable params() { new TestCase( "L1=1", "BwAAAPIe53BtV9PCAAABfhMmioA", + "BwAAALQa3_B5oSRHAAABfhMmioA", "KI4kVxcCLIMM2_VQGD575d-tm41vBpU_-TUExUU_bL3Puq_EBgIaLac", + "C8t2xhEnokpbS-rpcaL9_JYgRQ", "2022-01-01T01:00:00.000Z", b -> { b.field("@timestamp", "2022-01-01T01:00:00Z"); @@ -322,7 +358,9 @@ public static Iterable params() { new TestCase( "L1=min", "BwAAAAhu7hy1RoXRAAABfhMmioA", + "BwAAAHJevaZ42qO0AAABfhMmioA", "KI4kVxcCLIMM2_VQGD575d8caJ3TBpU_-cLpg-VnCBnhYk33HZBle6E", + "C-92gIC6326BrsHHzrPq6L6b7w", "2022-01-01T01:00:00.000Z", b -> { b.field("@timestamp", "2022-01-01T01:00:00Z"); @@ -335,7 +373,9 @@ public static Iterable params() { new TestCase( "L2=1234", "BwAAAATrNu7TTpc-AAABfhMmioA", + "BwAAAAykDD72vvVsAAABfhMmioA", "KI_1WxF60L0IczG5ftUCWdndcGtgBpU_-QfM2BaR0DMagIfw3TDu_mA", + "tCp2ar0mjlGW0gXveQToNzewqg", "2022-01-01T01:00:00.000Z", b -> { b.field("@timestamp", "2022-01-01T01:00:00Z"); @@ -348,7 +388,9 @@ public static Iterable params() { new TestCase( "o.L3=max", "BwAAAGBQI6THHqxoAAABfhMmioA", + "BwAAAMipdDqKd-XPAAABfhMmioA", "KN4a6QzKhzc3nwzNLuZkV51xxTOVBpU_-erUU1qSW4eJ0kP0RmAB9TE", + "S5l2EpF1iWUfxVvyegvENonWEw", "2022-01-01T01:00:00.000Z", b -> { b.field("@timestamp", "2022-01-01T01:00:00.000Z"); @@ -379,7 +421,9 @@ public static Iterable params() { new TestCase( "i1=1", "BwAAAEMS_RWRoHYjAAABfhMmioA", + "BwAAABKQqNW1IG5AAAABfhMmioA", "KLGFpvAV8QkWSmX54kXFMgitm41vBpU_-TUExUU_bL3Puq_EBgIaLac", + "nct2JZgFouaJFx9FPyphfxTTBw", "2022-01-01T01:00:00.000Z", b -> { b.field("@timestamp", "2022-01-01T01:00:00Z"); @@ -392,7 +436,9 @@ public static Iterable params() { new TestCase( "i1=min", "BwAAAKdlQM5ILoA1AAABfhMmioA", + "BwAAAFVne7eYMxr1AAABfhMmioA", "KLGFpvAV8QkWSmX54kXFMgjV8hFQBpU_-WG2MicRGWwJdBKWq2F4qy4", + "ncx27Tdr-Fw8YsGkFZT_X2isdg", "2022-01-01T01:00:00.000Z", b -> { b.field("@timestamp", "2022-01-01T01:00:00Z"); @@ -405,7 +451,9 @@ public static Iterable params() { new TestCase( "i2=1234", "BwAAALhxfB6J0kBFAAABfhMmioA", + "BwAAAOfBxmKrLd6MAAABfhMmioA", "KJc4-5eN1uAlYuAknQQLUlxavn2sBpU_-UEXBjgaH1uYcbayrOhdgpc", + "7RN22sJMTvFnwNcgZkI3oCIBvQ", "2022-01-01T01:00:00.000Z", b -> { b.field("@timestamp", "2022-01-01T01:00:00Z"); @@ -418,7 +466,9 @@ public static Iterable params() { new TestCase( "o.i3=max", "BwAAAOlxKf19CbfdAAABfhMmioA", + "BwAAAPasG-voJ30IAAABfhMmioA", "KKqnzPNBe8ObksSo8rNaIFPZPCcBBpU_-Rhd_U6Jn2pjQz2zpmBuJb4", + "ae1249zvwt9WR5M0TsFbT-_R4A", "2022-01-01T01:00:00.000Z", b -> { b.field("@timestamp", "2022-01-01T01:00:00Z"); @@ -449,7 +499,9 @@ public static Iterable params() { new TestCase( "s1=1", "BwAAAI_y-8kD_BFeAAABfhMmioA", + "BwAAAFXFZfzCD9-tAAABfhMmioA", "KFi_JDbvzWyAawmh8IEXedwGlT_5rZuNb-1ruHTTZhtsXRZpZRwWFoc", + "E3bLGR5i_E-AjHaQj6NSgLqXqQ", "2022-01-01T01:00:00.000Z", b -> { b.field("@timestamp", "2022-01-01T01:00:00Z"); @@ -462,7 +514,9 @@ public static Iterable params() { new TestCase( "s1=min", "BwAAAGV8VNVnmPVNAAABfhMmioA", + "BwAAAHMdz2yGuWQ5AAABfhMmioA", "KFi_JDbvzWyAawmh8IEXedwGlT_5JgBZj9BSCms2_jgeFFhsmDlNFdM", + "E3YA0Uj0mUMBIfe85M1eDvKJVg", "2022-01-01T01:00:00.000Z", b -> { b.field("@timestamp", "2022-01-01T01:00:00Z"); @@ -475,7 +529,9 @@ public static Iterable params() { new TestCase( "s2=1234", "BwAAAFO8mUr-J5CpAAABfhMmioA", + "BwAAAKYKsByk_H1KAAABfhMmioA", "KKEQ2p3CkpMH61hNk_SuvI0GlT_53XBrYP5TPdmCR-vREPnt20e9f9w", + "CHYqMfreOGDp369Up7wkbxfD7g", "2022-01-01T01:00:00.000Z", b -> { b.field("@timestamp", "2022-01-01T01:00:00Z"); @@ -486,9 +542,11 @@ public static Iterable params() { ); items.add( new TestCase( - "o.s3=max", + "o.s3=min", "BwAAAAKh6K11zWeuAAABfhMmioA", + "BwAAAJ00j5PXlZF8AAABfhMmioA", "KKVMoT_-GS95fvIBtR7XK9oGlT_5Dme9-H3sen0WZ7leJpCj7-vXau4", + "BXaV38EUXIsWps32nOam3wdX-g", "2022-01-01T01:00:00.000Z", b -> { b.field("@timestamp", "2022-01-01T01:00:00Z"); @@ -519,7 +577,9 @@ public static Iterable params() { new TestCase( "b1=1", "BwAAANKxqgT5JDQfAAABfhMmioA", + "BwAAACaAVo28B16ZAAABfhMmioA", "KGPAUhTjWOsRfDmYp3SUELatm41vBpU_-TUExUU_bL3Puq_EBgIaLac", + "Uct2hEdPEZMCnGZ0_NM3so6Z3w", "2022-01-01T01:00:00.000Z", b -> { b.field("@timestamp", "2022-01-01T01:00:00Z"); @@ -532,7 +592,9 @@ public static Iterable params() { new TestCase( "b1=min", "BwAAAN_PD--DgUvoAAABfhMmioA", + "BwAAAMkeTGox24cdAAABfhMmioA", "KGPAUhTjWOsRfDmYp3SUELYoK6qHBpU_-d8HkZFJ3aL2ZV1lgHAjT1g", + "USF2EOdV0G1EpXE-GDtFWr7jHg", "2022-01-01T01:00:00.000Z", b -> { b.field("@timestamp", "2022-01-01T01:00:00Z"); @@ -545,7 +607,9 @@ public static Iterable params() { new TestCase( "b2=12", "BwAAAKqX5QjiuhsEAAABfhMmioA", + "BwAAAB3kQls40KoVAAABfhMmioA", "KA58oUMzXeX1V5rh51Ste0K5K9vPBpU_-Wn8JQplO-x3CgoslYO5Vks", + "06J2CccOS3823wXT-Ntmaaz8nw", "2022-01-01T01:00:00.000Z", b -> { b.field("@timestamp", "2022-01-01T01:00:00Z"); @@ -558,7 +622,9 @@ public static Iterable params() { new TestCase( "o.s3=max", "BwAAAMJ4YtN_21XHAAABfhMmioA", + "BwAAAD25hXYPluEiAAABfhMmioA", "KIwZH-StJBobjk9tCV-0OgjKmuwGBpU_-Sd-SdnoH3sbfKLgse-briE", + "P5N2wn3fYvtVKKIOWB2n7AQweA", "2022-01-01T01:00:00.000Z", b -> { b.field("@timestamp", "2022-01-01T01:00:00Z"); @@ -589,7 +655,9 @@ public static Iterable params() { new TestCase( "ip1=192.168.0.1", "BwAAAD5km9raIz_rAAABfhMmioA", + "BwAAABJYg92lKoxdAAABfhMmioA", "KNj6cLPRNEkqdjfOPIbg0wULrOlWBpU_-efWDsz6B6AnnwbZ7GeeocE", + "3ft2E9wEcQ7qLhAhG-bgQxIC_w", "2022-01-01T01:00:00.000Z", b -> { b.field("@timestamp", "2022-01-01T01:00:00Z"); @@ -606,7 +674,9 @@ public static Iterable params() { new TestCase( "ip1=12.12.45.254", "BwAAAAWfEH_e_6wIAAABfhMmioA", + "BwAAAEqgz7x99nlVAAABfhMmioA", "KNj6cLPRNEkqdjfOPIbg0wVhJ08TBpU_-bANzLhvKPczlle7Pq0z8Qw", + "3RV2CkXc8fttjm4g1xCOxnlI7A", "2022-01-01T01:00:00.000Z", b -> { b.field("@timestamp", "2022-01-01T01:00:00Z"); @@ -623,7 +693,9 @@ public static Iterable params() { new TestCase( "ip2=FE80:CD00:0000:0CDE:1257:0000:211E:729C", "BwAAAGrrLHr1O4iQAAABfhMmioA", + "BwAAAHTrphGgaaxEAAABfhMmioA", "KNDo3zGxO9HfN9XYJwKw2Z20h-WsBpU_-f4dSOLGSRlL1hoY2mgERuo", + "4ch2pulC-JV1tGryTUWFGMsHdQ", "2022-01-01T01:00:00.000Z", b -> { b.field("@timestamp", "2022-01-01T01:00:00Z"); @@ -636,7 +708,9 @@ public static Iterable params() { new TestCase( "o.ip3=2001:db8:85a3:8d3:1319:8a2e:370:7348", "BwAAAK7d-9aKOS1MAAABfhMmioA", + "BwAAANak4OD67dUVAAABfhMmioA", "KLXDcBBWJAjgJvjSdF_EJwraAQUzBpU_-ba6HZsIyKnGcbmc3KRLlmI", + "ZKJ28_e_PHW9GEdDVaEHOBblSw", "2022-01-01T01:00:00.000Z", b -> { b.field("@timestamp", "2022-01-01T01:00:00Z"); @@ -667,7 +741,9 @@ public static Iterable params() { new TestCase( "huge", "BwAAAPdECvXBSl3xAAABfhMmioA", + "BwAAAEDsUEqRn2O2AAABfhMmioA", "LIe18i0rRU_Bt9vB82F46LaS9mrUkvZq1K_2Gi7UEFMhFwNXrLA_H8TLpUr4", + "tIaGaQ90mXYrnllFi07m7A2pRMI", "2022-01-01T01:00:00.000Z", b -> { b.field("@timestamp", "2022-01-01T01:00:00Z"); @@ -689,18 +765,32 @@ public TsidExtractingIdFieldMapperTests(@Named("testCase") TestCase testCase) { this.testCase = testCase; } - public void testExpectedId() throws IOException { - assertThat(parse(mapperService(), testCase.source).id(), equalTo(testCase.expectedId)); + public void testExpectedIdWithRoutingPath() throws IOException { + assertThat(parse(mapperService(false), testCase.source).id(), equalTo(testCase.expectedIdWithRoutingPath)); } - public void testProvideExpectedId() throws IOException { - assertThat(parse(testCase.expectedId, mapperService(), testCase.source).id(), equalTo(testCase.expectedId)); + public void testExpectedIdWithIndexDimensions() throws IOException { + assertThat(parse(mapperService(true), testCase.source).id(), equalTo(testCase.expectedIdWithIndexDimensions)); + } + + public void testProvideExpectedIdWithRoutingPath() throws IOException { + assertThat( + parse(testCase.expectedIdWithRoutingPath, mapperService(false), testCase.source).id(), + equalTo(testCase.expectedIdWithRoutingPath) + ); + } + + public void testProvideExpectedIdWithIndexDimensions() throws IOException { + assertThat( + parse(testCase.expectedIdWithIndexDimensions, mapperService(true), testCase.source).id(), + equalTo(testCase.expectedIdWithIndexDimensions) + ); } - public void testEquivalentSources() throws IOException { - MapperService mapperService = mapperService(); + public void testEquivalentSourcesWithRoutingPath() throws IOException { + MapperService mapperService = mapperService(false); for (CheckedConsumer equivalent : testCase.equivalentSources) { - assertThat(parse(mapperService, equivalent).id(), equalTo(testCase.expectedId)); + assertThat(parse(mapperService, equivalent).id(), equalTo(testCase.expectedIdWithRoutingPath)); } } @@ -724,26 +814,35 @@ private ParsedDocument parse(@Nullable String id, MapperService mapperService, C } } - public void testRoutingPathCompliant() throws IOException { - byte[] bytes = Base64.getUrlDecoder().decode(testCase.expectedId); + public void testRoutingHashCompliantWithRoutingPath() throws IOException { + byte[] bytes = Base64.getUrlDecoder().decode(testCase.expectedIdWithRoutingPath); assertEquals(ROUTING_HASH, ByteUtils.readIntLE(bytes, 0)); } - private Settings indexSettings(IndexVersion version) { - return Settings.builder() + public void testRoutingHashCompliantWithIndexDimensions() throws IOException { + byte[] bytes = Base64.getUrlDecoder().decode(testCase.expectedIdWithIndexDimensions); + assertEquals(ROUTING_HASH, ByteUtils.readIntLE(bytes, 0)); + } + + private Settings indexSettings(IndexVersion version, boolean indexDimensions) { + Settings.Builder builder = Settings.builder() .put(IndexSettings.MODE.getKey(), "time_series") .put(IndexMetadata.SETTING_VERSION_CREATED, version) .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, between(1, 100)) .put(IndexSettings.TIME_SERIES_START_TIME.getKey(), "-9999-01-01T00:00:00Z") .put(IndexSettings.TIME_SERIES_END_TIME.getKey(), "9999-01-01T00:00:00Z") - .put(IndexMetadata.INDEX_ROUTING_PATH.getKey(), "r1,r2,o.r3") - .put(MapperService.INDEX_MAPPING_DIMENSION_FIELDS_LIMIT_SETTING.getKey(), 100) - .build(); + .put(MapperService.INDEX_MAPPING_DIMENSION_FIELDS_LIMIT_SETTING.getKey(), 100); + if (indexDimensions) { + builder.put(IndexMetadata.INDEX_DIMENSIONS.getKey(), "r1,r2,k1,k2,L1,L2,i1,i2,s1,s2,b1,b2,ip1,ip2,o.*"); + } else { + builder.put(IndexMetadata.INDEX_ROUTING_PATH.getKey(), "r1,r2,o.r3"); + } + return builder.build(); } - private MapperService mapperService() throws IOException { + private MapperService mapperService(boolean indexDimensions) throws IOException { IndexVersion version = IndexVersionUtils.randomCompatibleVersion(random()); - return createMapperService(indexSettings(version), mapping(b -> { + return createMapperService(indexSettings(version, indexDimensions), mapping(b -> { b.startObject("r1").field("type", "keyword").field("time_series_dimension", true).endObject(); b.startObject("r2").field("type", "keyword").field("time_series_dimension", true).endObject(); b.startObject("k1").field("type", "keyword").field("time_series_dimension", true).endObject(); @@ -785,29 +884,61 @@ protected boolean isConfigurable() { @Override protected void registerParameters(ParameterChecker checker) throws IOException {} - public void testSourceDescription() throws IOException { - assertThat(TsidExtractingIdFieldMapper.INSTANCE.documentDescription(documentParserContext()), equalTo("a time series document")); - ParsedDocument d = parse(mapperService(), testCase.randomSource()); + public void testSourceDescriptionWithRoutingPath() throws IOException { + assertThat( + TsidExtractingIdFieldMapper.INSTANCE.documentDescription(documentParserContext(false)), + equalTo("a time series document") + ); + ParsedDocument d = parse(mapperService(false), testCase.randomSource()); + IndexableField timestamp = d.rootDoc().getField(DataStreamTimestampFieldMapper.DEFAULT_PATH); + assertThat( + TsidExtractingIdFieldMapper.INSTANCE.documentDescription(documentParserContext(false, timestamp)), + equalTo("a time series document at [" + testCase.expectedTimestamp + "]") + ); + IndexableField tsid = d.rootDoc().getField(TimeSeriesIdFieldMapper.NAME); + assertThat( + TsidExtractingIdFieldMapper.INSTANCE.documentDescription(documentParserContext(false, tsid)), + equalTo("a time series document with tsid " + testCase.expectedTsidWithRoutingPath) + ); + assertThat( + TsidExtractingIdFieldMapper.INSTANCE.documentDescription(documentParserContext(false, tsid, timestamp)), + equalTo("a time series document with tsid " + testCase.expectedTsidWithRoutingPath + " at [" + testCase.expectedTimestamp + "]") + ); + } + + public void testSourceDescriptionWithIndexDimensions() throws IOException { + assertThat( + TsidExtractingIdFieldMapper.INSTANCE.documentDescription(documentParserContext(false)), + equalTo("a time series document") + ); + ParsedDocument d = parse(mapperService(false), testCase.randomSource()); IndexableField timestamp = d.rootDoc().getField(DataStreamTimestampFieldMapper.DEFAULT_PATH); assertThat( - TsidExtractingIdFieldMapper.INSTANCE.documentDescription(documentParserContext(timestamp)), + TsidExtractingIdFieldMapper.INSTANCE.documentDescription(documentParserContext(false, timestamp)), equalTo("a time series document at [" + testCase.expectedTimestamp + "]") ); IndexableField tsid = d.rootDoc().getField(TimeSeriesIdFieldMapper.NAME); assertThat( - TsidExtractingIdFieldMapper.INSTANCE.documentDescription(documentParserContext(tsid)), - equalTo("a time series document with tsid " + testCase.expectedTsid) + TsidExtractingIdFieldMapper.INSTANCE.documentDescription(documentParserContext(false, tsid)), + equalTo("a time series document with tsid " + testCase.expectedTsidWithRoutingPath) ); assertThat( - TsidExtractingIdFieldMapper.INSTANCE.documentDescription(documentParserContext(tsid, timestamp)), - equalTo("a time series document with tsid " + testCase.expectedTsid + " at [" + testCase.expectedTimestamp + "]") + TsidExtractingIdFieldMapper.INSTANCE.documentDescription(documentParserContext(false, tsid, timestamp)), + equalTo("a time series document with tsid " + testCase.expectedTsidWithRoutingPath + " at [" + testCase.expectedTimestamp + "]") ); } - private TestDocumentParserContext documentParserContext(IndexableField... fields) throws IOException { + private TestDocumentParserContext documentParserContext(boolean indexDimensions, IndexableField... fields) throws IOException { + CheckedConsumer source; + // not using a random source here as the index.dimensions id is sensitive to how ips are represented (e.g. equivalent ipv4 vs ipv6) + if (indexDimensions) { + source = testCase.source; + } else { + source = testCase.randomSource(); + } TestDocumentParserContext ctx = new TestDocumentParserContext( - mapperService().mappingLookup(), - source(null, testCase.randomSource(), null) + mapperService(indexDimensions).mappingLookup(), + source(null, source, null) ); for (IndexableField f : fields) { ctx.doc().add(f); @@ -815,10 +946,34 @@ private TestDocumentParserContext documentParserContext(IndexableField... fields return ctx; } - public void testParsedDescription() throws IOException { + public void testParsedDescriptionWithRoutingPath() throws IOException { assertThat( - TsidExtractingIdFieldMapper.INSTANCE.documentDescription(parse(mapperService(), testCase.randomSource())), - equalTo("[" + testCase.expectedId + "][" + testCase.expectedTsid + "@" + testCase.expectedTimestamp + "]") + TsidExtractingIdFieldMapper.INSTANCE.documentDescription(parse(mapperService(false), testCase.randomSource())), + equalTo( + "[" + + testCase.expectedIdWithRoutingPath + + "][" + + testCase.expectedTsidWithRoutingPath + + "@" + + testCase.expectedTimestamp + + "]" + ) + ); + } + + public void testParsedDescriptionWithIndexDimensions() throws IOException { + // not using a random source here as the index.dimensions id is sensitive to how ips are represented (e.g. equivalent ipv4 vs ipv6) + assertThat( + TsidExtractingIdFieldMapper.INSTANCE.documentDescription(parse(mapperService(true), testCase.source)), + equalTo( + "[" + + testCase.expectedIdWithIndexDimensions + + "][" + + testCase.expectedTsidWithIndexDimensions + + "@" + + testCase.expectedTimestamp + + "]" + ) ); } } diff --git a/test/framework/src/main/java/org/elasticsearch/test/InternalTestCluster.java b/test/framework/src/main/java/org/elasticsearch/test/InternalTestCluster.java index 8b4d29396e5d5..9e13105994de6 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/InternalTestCluster.java +++ b/test/framework/src/main/java/org/elasticsearch/test/InternalTestCluster.java @@ -26,6 +26,7 @@ import org.elasticsearch.action.admin.cluster.node.stats.NodeStats; import org.elasticsearch.action.admin.indices.stats.CommonStatsFlags; import org.elasticsearch.action.admin.indices.stats.CommonStatsFlags.Flag; +import org.elasticsearch.action.index.IndexRequest; import org.elasticsearch.action.support.DestructiveOperations; import org.elasticsearch.action.support.RefCountingRunnable; import org.elasticsearch.action.support.replication.TransportReplicationAction; @@ -2444,7 +2445,7 @@ synchronized String routingKeyForShard(Index index, int shard, Random random) { IndexRouting indexRouting = IndexRouting.fromIndexMetadata(clusterState.metadata().getProject().getIndexSafe(index)); while (true) { String routing = RandomStrings.randomAsciiLettersOfLength(random, 10); - if (shard == indexRouting.indexShard("id", routing, null, null, null)) { + if (shard == indexRouting.indexShard(new IndexRequest().id("id").routing(routing))) { return routing; } }