Skip to content

Commit 205466f

Browse files
felixbarnyfelixbarny
committed
Refactor IndexRouting.ExtractFromSource to be an abstract class
With implementations IndexRouting.ExtractFromSource.ForRoutingPath and IndexRouting.ExtractFromSource.ForIndexDimensions. This addresses review comments from elastic#132566.
1 parent fe4c41e commit 205466f

File tree

12 files changed

+308
-260
lines changed

12 files changed

+308
-260
lines changed

‎server/src/main/java/org/elasticsearch/action/index/IndexRequest.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -923,7 +923,7 @@ public Index getConcreteWriteIndex(IndexAbstraction ia, ProjectMetadata project)
923923

924924
@Override
925925
public int route(IndexRouting indexRouting) {
926-
return indexRouting.indexShard(id, routing, tsid, indexSource.contentType(), indexSource.bytes());
926+
return indexRouting.indexShard(this);
927927
}
928928

929929
public IndexRequest setRequireAlias(boolean requireAlias) {

‎server/src/main/java/org/elasticsearch/cluster/routing/IndexRouting.java‎

Lines changed: 114 additions & 208 deletions
Large diffs are not rendered by default.

‎server/src/main/java/org/elasticsearch/cluster/routing/RoutingHashBuilder.java‎

Lines changed: 135 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,135 @@
1+
/*
2+
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
3+
* or more contributor license agreements. Licensed under the "Elastic License
4+
* 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
5+
* Public License v 1"; you may not use this file except in compliance with, at
6+
* your election, the "Elastic License 2.0", the "GNU Affero General Public
7+
* License v3.0 only", or the "Server Side Public License, v 1".
8+
*/
9+
10+
package org.elasticsearch.cluster.routing;
11+
12+
import org.apache.lucene.util.BytesRef;
13+
import org.elasticsearch.common.ParsingException;
14+
import org.elasticsearch.common.Strings;
15+
import org.elasticsearch.common.util.ByteUtils;
16+
import org.elasticsearch.core.Nullable;
17+
import org.elasticsearch.index.IndexVersions;
18+
import org.elasticsearch.xcontent.XContentParser;
19+
import org.elasticsearch.xcontent.XContentString;
20+
21+
import java.io.IOException;
22+
import java.util.ArrayList;
23+
import java.util.Collections;
24+
import java.util.List;
25+
import java.util.function.IntSupplier;
26+
import java.util.function.Predicate;
27+
28+
import static org.elasticsearch.common.xcontent.XContentParserUtils.ensureExpectedToken;
29+
import static org.elasticsearch.common.xcontent.XContentParserUtils.expectValueToken;
30+
31+
/**
32+
* A builder for computing a hash from fields in the document source that are part of the
33+
* {@link org.elasticsearch.cluster.metadata.IndexMetadata#INDEX_ROUTING_PATH}.
34+
* It is used in the context of {@link IndexRouting.ExtractFromSource.ForRoutingPath} to determine the shard a document should be routed to.
35+
*/
36+
public class RoutingHashBuilder {
37+
private final List<NameAndHash> hashes = new ArrayList<>();
38+
private final Predicate<String> isRoutingPath;
39+
40+
public RoutingHashBuilder(Predicate<String> isRoutingPath) {
41+
this.isRoutingPath = isRoutingPath;
42+
}
43+
44+
public void addMatching(String fieldName, BytesRef string) {
45+
if (isRoutingPath.test(fieldName)) {
46+
addHash(fieldName, string);
47+
}
48+
}
49+
50+
/**
51+
* Only expected to be called for old indices created before
52+
* {@link IndexVersions#TIME_SERIES_ROUTING_HASH_IN_ID} while creating (during ingestion)
53+
* or synthesizing (at query time) the _id field.
54+
*/
55+
public String createId(byte[] suffix, IntSupplier onEmpty) {
56+
byte[] idBytes = new byte[4 + suffix.length];
57+
ByteUtils.writeIntLE(buildHash(onEmpty), idBytes, 0);
58+
System.arraycopy(suffix, 0, idBytes, 4, suffix.length);
59+
return Strings.BASE_64_NO_PADDING_URL_ENCODER.encodeToString(idBytes);
60+
}
61+
62+
void extractObject(@Nullable String path, XContentParser source) throws IOException {
63+
while (source.currentToken() != XContentParser.Token.END_OBJECT) {
64+
ensureExpectedToken(XContentParser.Token.FIELD_NAME, source.currentToken(), source);
65+
String fieldName = source.currentName();
66+
String subPath = path == null ? fieldName : path + "." + fieldName;
67+
source.nextToken();
68+
extractItem(subPath, source);
69+
}
70+
}
71+
72+
private void extractArray(@Nullable String path, XContentParser source) throws IOException {
73+
while (source.currentToken() != XContentParser.Token.END_ARRAY) {
74+
expectValueToken(source.currentToken(), source);
75+
extractItem(path, source);
76+
}
77+
}
78+
79+
private void extractItem(String path, XContentParser source) throws IOException {
80+
switch (source.currentToken()) {
81+
case START_OBJECT:
82+
source.nextToken();
83+
extractObject(path, source);
84+
source.nextToken();
85+
break;
86+
case VALUE_STRING:
87+
case VALUE_NUMBER:
88+
case VALUE_BOOLEAN:
89+
XContentString.UTF8Bytes utf8Bytes = source.optimizedText().bytes();
90+
addHash(path, new BytesRef(utf8Bytes.bytes(), utf8Bytes.offset(), utf8Bytes.length()));
91+
source.nextToken();
92+
break;
93+
case START_ARRAY:
94+
source.nextToken();
95+
extractArray(path, source);
96+
source.nextToken();
97+
break;
98+
case VALUE_NULL:
99+
source.nextToken();
100+
break;
101+
default:
102+
throw new ParsingException(
103+
source.getTokenLocation(),
104+
"Cannot extract routing path due to unexpected token [{}]",
105+
source.currentToken()
106+
);
107+
}
108+
}
109+
110+
private void addHash(String path, BytesRef value) {
111+
hashes.add(new NameAndHash(new BytesRef(path), IndexRouting.ExtractFromSource.hash(value), hashes.size()));
112+
}
113+
114+
int buildHash(IntSupplier onEmpty) {
115+
if (hashes.isEmpty()) {
116+
return onEmpty.getAsInt();
117+
}
118+
Collections.sort(hashes);
119+
int hash = 0;
120+
for (NameAndHash nah : hashes) {
121+
hash = 31 * hash + (IndexRouting.ExtractFromSource.hash(nah.name) ^ nah.hash);
122+
}
123+
return hash;
124+
}
125+
126+
private record NameAndHash(BytesRef name, int hash, int order) implements Comparable<NameAndHash> {
127+
@Override
128+
public int compareTo(NameAndHash o) {
129+
int i = name.compareTo(o.name);
130+
if (i != 0) return i;
131+
// ensures array values are in the order as they appear in the source
132+
return Integer.compare(order, o.order);
133+
}
134+
}
135+
}

‎server/src/main/java/org/elasticsearch/index/IndexMode.java‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -223,7 +223,8 @@ public RoutingFields buildRoutingFields(IndexSettings settings, SourceToParse so
223223
// If the source already has a _tsid field, we don't need to extract routing from the source.
224224
return RoutingFields.Noop.INSTANCE;
225225
}
226-
IndexRouting.ExtractFromSource routing = (IndexRouting.ExtractFromSource) settings.getIndexRouting();
226+
IndexRouting.ExtractFromSource.ForRoutingPath routing = (IndexRouting.ExtractFromSource.ForRoutingPath) settings
227+
.getIndexRouting();
227228
return new RoutingPathFields(routing.builder());
228229
}
229230

‎server/src/main/java/org/elasticsearch/index/mapper/IdLoader.java‎

Lines changed: 6 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@
1717
import org.apache.lucene.util.BytesRef;
1818
import org.elasticsearch.cluster.metadata.DataStream;
1919
import org.elasticsearch.cluster.routing.IndexRouting;
20+
import org.elasticsearch.cluster.routing.RoutingHashBuilder;
2021
import org.elasticsearch.index.fieldvisitor.LeafStoredFieldLoader;
2122

2223
import java.io.IOException;
@@ -37,7 +38,7 @@ static IdLoader fromLeafStoredFieldLoader() {
3738
/**
3839
* @return returns an {@link IdLoader} instance that syn synthesizes _id from routing, _tsid and @timestamp fields.
3940
*/
40-
static IdLoader createTsIdLoader(IndexRouting.ExtractFromSource indexRouting, List<String> routingPaths) {
41+
static IdLoader createTsIdLoader(IndexRouting.ExtractFromSource.ForRoutingPath indexRouting, List<String> routingPaths) {
4142
return new TsIdLoader(indexRouting, routingPaths);
4243
}
4344

@@ -58,19 +59,19 @@ sealed interface Leaf permits StoredLeaf, TsIdLeaf {
5859

5960
final class TsIdLoader implements IdLoader {
6061

61-
private final IndexRouting.ExtractFromSource indexRouting;
62+
private final IndexRouting.ExtractFromSource.ForRoutingPath indexRouting;
6263
private final List<String> routingPaths;
6364

64-
TsIdLoader(IndexRouting.ExtractFromSource indexRouting, List<String> routingPaths) {
65+
TsIdLoader(IndexRouting.ExtractFromSource.ForRoutingPath indexRouting, List<String> routingPaths) {
6566
this.routingPaths = routingPaths;
6667
this.indexRouting = indexRouting;
6768
}
6869

6970
public IdLoader.Leaf leaf(LeafStoredFieldLoader loader, LeafReader reader, int[] docIdsInLeaf) throws IOException {
70-
IndexRouting.ExtractFromSource.RoutingHashBuilder[] builders = null;
71+
RoutingHashBuilder[] builders = null;
7172
if (indexRouting != null) {
7273
// this branch is for legacy indices before IndexVersions.TIME_SERIES_ROUTING_HASH_IN_ID
73-
builders = new IndexRouting.ExtractFromSource.RoutingHashBuilder[docIdsInLeaf.length];
74+
builders = new RoutingHashBuilder[docIdsInLeaf.length];
7475
for (int i = 0; i < builders.length; i++) {
7576
builders[i] = indexRouting.builder();
7677
}

‎server/src/main/java/org/elasticsearch/index/mapper/RoutingPathFields.java‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,7 @@
1111

1212
import org.apache.lucene.util.BytesRef;
1313
import org.apache.lucene.util.StringHelper;
14-
import org.elasticsearch.cluster.routing.IndexRouting;
14+
import org.elasticsearch.cluster.routing.RoutingHashBuilder;
1515
import org.elasticsearch.common.bytes.BytesArray;
1616
import org.elasticsearch.common.bytes.BytesReference;
1717
import org.elasticsearch.common.hash.Murmur3Hasher;
@@ -59,17 +59,17 @@ public final class RoutingPathFields implements RoutingFields {
5959
* Builds the routing. Used for building {@code _id}. If null then skipped.
6060
*/
6161
@Nullable
62-
private final IndexRouting.ExtractFromSource.RoutingHashBuilder routingBuilder;
62+
private final RoutingHashBuilder routingBuilder;
6363

64-
public RoutingPathFields(@Nullable IndexRouting.ExtractFromSource.RoutingHashBuilder routingBuilder) {
64+
public RoutingPathFields(@Nullable RoutingHashBuilder routingBuilder) {
6565
this.routingBuilder = routingBuilder;
6666
}
6767

6868
SortedMap<BytesRef, List<BytesReference>> routingValues() {
6969
return Collections.unmodifiableSortedMap(routingValues);
7070
}
7171

72-
IndexRouting.ExtractFromSource.RoutingHashBuilder routingBuilder() {
72+
RoutingHashBuilder routingBuilder() {
7373
return routingBuilder;
7474
}
7575

‎server/src/main/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapper.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,7 @@
1414
import org.apache.lucene.document.StringField;
1515
import org.apache.lucene.search.Query;
1616
import org.apache.lucene.util.BytesRef;
17-
import org.elasticsearch.cluster.routing.IndexRouting;
17+
import org.elasticsearch.cluster.routing.RoutingHashBuilder;
1818
import org.elasticsearch.common.Strings;
1919
import org.elasticsearch.common.bytes.BytesReference;
2020
import org.elasticsearch.common.io.stream.BytesStreamOutput;
@@ -175,7 +175,7 @@ public void postParse(DocumentParserContext context) throws IOException {
175175
context.doc().add(new SortedDocValuesField(fieldType().name(), timeSeriesId));
176176
}
177177

178-
IndexRouting.ExtractFromSource.RoutingHashBuilder routingBuilder;
178+
RoutingHashBuilder routingBuilder;
179179
if (getIndexVersionCreated(context).before(IndexVersions.TIME_SERIES_ROUTING_HASH_IN_ID) && routingPathFields != null) {
180180
// For legacy indices, we need to create the routing hash from the routing path fields.
181181
routingBuilder = routingPathFields.routingBuilder();

‎server/src/main/java/org/elasticsearch/index/mapper/TsidExtractingIdFieldMapper.java‎

Lines changed: 6 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@
1414
import org.apache.lucene.index.IndexableField;
1515
import org.apache.lucene.util.BytesRef;
1616
import org.elasticsearch.cluster.routing.IndexRouting;
17+
import org.elasticsearch.cluster.routing.RoutingHashBuilder;
1718
import org.elasticsearch.common.Strings;
1819
import org.elasticsearch.common.hash.MurmurHash3;
1920
import org.elasticsearch.common.hash.MurmurHash3.Hash128;
@@ -46,11 +47,7 @@ public IndexFieldData.Builder fielddataBuilder(FieldDataContext fieldDataContext
4647

4748
private static final long SEED = 0;
4849

49-
public static BytesRef createField(
50-
DocumentParserContext context,
51-
IndexRouting.ExtractFromSource.RoutingHashBuilder routingBuilder,
52-
BytesRef tsid
53-
) {
50+
public static BytesRef createField(DocumentParserContext context, RoutingHashBuilder routingBuilder, BytesRef tsid) {
5451
final long timestamp = DataStreamTimestampFieldMapper.extractTimestampValue(context.doc());
5552
String id;
5653
if (routingBuilder != null) {
@@ -65,7 +62,9 @@ public static BytesRef createField(
6562
* at all we just skip the assertion because we can't be sure
6663
* it always must pass.
6764
*/
68-
IndexRouting.ExtractFromSource indexRouting = (IndexRouting.ExtractFromSource) context.indexSettings().getIndexRouting();
65+
IndexRouting.ExtractFromSource.ForRoutingPath indexRouting = (IndexRouting.ExtractFromSource.ForRoutingPath) context
66+
.indexSettings()
67+
.getIndexRouting();
6968
assert context.getDynamicMappers().isEmpty() == false
7069
|| context.getDynamicRuntimeFields().isEmpty() == false
7170
|| id.equals(indexRouting.createId(context.sourceToParse().getXContentType(), context.sourceToParse().source(), suffix));
@@ -115,7 +114,7 @@ public static String createId(int routingHash, BytesRef tsid, long timestamp) {
115114

116115
public static String createId(
117116
boolean dynamicMappersExists,
118-
IndexRouting.ExtractFromSource.RoutingHashBuilder routingBuilder,
117+
RoutingHashBuilder routingBuilder,
119118
BytesRef tsid,
120119
long timestamp,
121120
byte[] suffix

‎server/src/main/java/org/elasticsearch/search/DefaultSearchContext.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -951,10 +951,10 @@ public SourceLoader newSourceLoader(@Nullable SourceFilter filter) {
951951
@Override
952952
public IdLoader newIdLoader() {
953953
if (indexService.getIndexSettings().getMode() == IndexMode.TIME_SERIES) {
954-
IndexRouting.ExtractFromSource indexRouting = null;
954+
IndexRouting.ExtractFromSource.ForRoutingPath indexRouting = null;
955955
List<String> routingPaths = null;
956956
if (indexService.getIndexSettings().getIndexVersionCreated().before(IndexVersions.TIME_SERIES_ROUTING_HASH_IN_ID)) {
957-
indexRouting = (IndexRouting.ExtractFromSource) indexService.getIndexSettings().getIndexRouting();
957+
indexRouting = (IndexRouting.ExtractFromSource.ForRoutingPath) indexService.getIndexSettings().getIndexRouting();
958958
routingPaths = indexService.getMetadata().getRoutingPaths();
959959
for (String routingField : routingPaths) {
960960
if (routingField.contains("*")) {

0 commit comments

Comments
 (0)