Skip to content

Commit c2286e1

Browse files
jordan-powersjordan-powers
authored
Optimize sort settings default providers (#138711)
This PR optimizes the default providers for the sort settings index.sort.field, index.sort.order, index.sort.mode, and index.sort.missing to fix a performance regression in the many-shards-quantitative benchmark associated with the introduction of these default providers in #135886.
1 parent d6a7761 commit c2286e1

File tree

5 files changed

+281
-70
lines changed

5 files changed

+281
-70
lines changed

‎benchmarks/build.gradle‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -48,6 +48,7 @@ dependencies {
4848
api(project(':x-pack:plugin:esql'))
4949
api(project(':x-pack:plugin:esql:compute'))
5050
api(project(':x-pack:plugin:mapper-exponential-histogram'))
51+
api(project(':x-pack:plugin:logsdb'))
5152
implementation project(path: ':libs:native')
5253
implementation project(path: ':libs:simdvec')
5354
implementation project(path: ':libs:exponential-histogram')

‎benchmarks/src/main/java/org/elasticsearch/benchmark/indices/common/MappingParsingBenchmark.java‎

Lines changed: 171 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,171 @@
1+
/*
2+
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
3+
* or more contributor license agreements. Licensed under the "Elastic License
4+
* 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
5+
* Public License v 1"; you may not use this file except in compliance with, at
6+
* your election, the "Elastic License 2.0", the "GNU Affero General Public
7+
* License v3.0 only", or the "Server Side Public License, v 1".
8+
*/
9+
10+
package org.elasticsearch.benchmark.indices.common;
11+
12+
import org.elasticsearch.TransportVersion;
13+
import org.elasticsearch.cluster.ClusterModule;
14+
import org.elasticsearch.cluster.metadata.IndexMetadata;
15+
import org.elasticsearch.common.compress.CompressedXContent;
16+
import org.elasticsearch.common.logging.LogConfigurator;
17+
import org.elasticsearch.common.settings.IndexScopedSettings;
18+
import org.elasticsearch.common.settings.Setting;
19+
import org.elasticsearch.common.settings.Settings;
20+
import org.elasticsearch.common.xcontent.LoggingDeprecationHandler;
21+
import org.elasticsearch.index.IndexSettings;
22+
import org.elasticsearch.index.IndexVersion;
23+
import org.elasticsearch.index.analysis.IndexAnalyzers;
24+
import org.elasticsearch.index.cache.bitset.BitsetFilterCache;
25+
import org.elasticsearch.index.mapper.MapperMetrics;
26+
import org.elasticsearch.index.mapper.MapperRegistry;
27+
import org.elasticsearch.index.mapper.MapperService;
28+
import org.elasticsearch.index.mapper.ProvidedIdFieldMapper;
29+
import org.elasticsearch.index.similarity.SimilarityService;
30+
import org.elasticsearch.indices.IndicesModule;
31+
import org.elasticsearch.script.Script;
32+
import org.elasticsearch.script.ScriptCompiler;
33+
import org.elasticsearch.script.ScriptContext;
34+
import org.elasticsearch.xcontent.NamedXContentRegistry;
35+
import org.elasticsearch.xcontent.XContentParserConfiguration;
36+
import org.elasticsearch.xpack.logsdb.LogsDBPlugin;
37+
import org.openjdk.jmh.annotations.Benchmark;
38+
import org.openjdk.jmh.annotations.BenchmarkMode;
39+
import org.openjdk.jmh.annotations.Fork;
40+
import org.openjdk.jmh.annotations.Measurement;
41+
import org.openjdk.jmh.annotations.Mode;
42+
import org.openjdk.jmh.annotations.OutputTimeUnit;
43+
import org.openjdk.jmh.annotations.Param;
44+
import org.openjdk.jmh.annotations.Scope;
45+
import org.openjdk.jmh.annotations.Setup;
46+
import org.openjdk.jmh.annotations.State;
47+
import org.openjdk.jmh.annotations.Warmup;
48+
49+
import java.io.IOException;
50+
import java.util.ArrayList;
51+
import java.util.HashSet;
52+
import java.util.List;
53+
import java.util.Map;
54+
import java.util.Random;
55+
import java.util.Set;
56+
import java.util.concurrent.TimeUnit;
57+
58+
@Fork(value = 1)
59+
@Warmup(iterations = 2)
60+
@Measurement(iterations = 5)
61+
@BenchmarkMode(Mode.AverageTime)
62+
@OutputTimeUnit(TimeUnit.MILLISECONDS)
63+
@State(Scope.Benchmark)
64+
public class MappingParsingBenchmark {
65+
static {
66+
// For Elasticsearch900Lucene101Codec:
67+
LogConfigurator.loadLog4jPlugins();
68+
LogConfigurator.configureESLogging();
69+
LogConfigurator.setNodeName("test");
70+
}
71+
72+
private static final String MAPPING = """
73+
{
74+
"_doc": {
75+
"dynamic": false,
76+
"properties": {
77+
"@timestamp": {
78+
"type": "date"
79+
},
80+
"host": {
81+
"type": "object",
82+
"properties": {
83+
"name": {
84+
"type": "keyword"
85+
}
86+
}
87+
},
88+
"message": {
89+
"type": "pattern_text"
90+
}
91+
}
92+
}
93+
}
94+
\s""";
95+
96+
@Param("1024")
97+
private int numIndices;
98+
99+
private List<MapperService> mapperServices;
100+
private CompressedXContent compressedMapping;
101+
102+
private Random random = new Random();
103+
private static final String CHARS = "abcdefghijklmnopqrstuvwxyz1234567890";
104+
105+
private String randomIndexName() {
106+
StringBuilder b = new StringBuilder();
107+
for (int i = 0; i < 10; i++) {
108+
b.append(CHARS.charAt(random.nextInt(CHARS.length())));
109+
}
110+
return b.toString();
111+
}
112+
113+
@Setup
114+
public void setUp() throws IOException {
115+
Settings settings = Settings.builder()
116+
.put("index.number_of_replicas", 0)
117+
.put("index.number_of_shards", 1)
118+
.put(IndexMetadata.SETTING_VERSION_CREATED, IndexVersion.current())
119+
.put("index.mode", "logsdb")
120+
.put("index.logsdb.sort_on_host_name", true)
121+
.put("index.logsdb.sort_on_message_template", true)
122+
.build();
123+
124+
LogsDBPlugin logsDBPlugin = new LogsDBPlugin(settings);
125+
126+
Set<Setting<?>> definedSettings = new HashSet<>(IndexScopedSettings.BUILT_IN_INDEX_SETTINGS);
127+
definedSettings.addAll(logsDBPlugin.getSettings().stream().filter(Setting::hasIndexScope).toList());
128+
IndexScopedSettings indexScopedSettings = new IndexScopedSettings(Settings.EMPTY, definedSettings);
129+
130+
mapperServices = new ArrayList<>(numIndices);
131+
for (int i = 0; i < numIndices; i++) {
132+
IndexMetadata meta = IndexMetadata.builder(randomIndexName()).settings(settings).build();
133+
IndexSettings indexSettings = new IndexSettings(meta, settings, indexScopedSettings);
134+
MapperRegistry mapperRegistry = new IndicesModule(List.of(logsDBPlugin)).getMapperRegistry();
135+
SimilarityService similarityService = new SimilarityService(indexSettings, null, Map.of());
136+
BitsetFilterCache bitsetFilterCache = new BitsetFilterCache(indexSettings, BitsetFilterCache.Listener.NOOP);
137+
MapperService mapperService = new MapperService(
138+
() -> TransportVersion.current(),
139+
indexSettings,
140+
IndexAnalyzers.of(Map.of()),
141+
XContentParserConfiguration.EMPTY.withRegistry(new NamedXContentRegistry(ClusterModule.getNamedXWriteables()))
142+
.withDeprecationHandler(LoggingDeprecationHandler.INSTANCE),
143+
similarityService,
144+
mapperRegistry,
145+
() -> {
146+
throw new UnsupportedOperationException();
147+
},
148+
new ProvidedIdFieldMapper(() -> true),
149+
new ScriptCompiler() {
150+
@Override
151+
public <T> T compile(Script script, ScriptContext<T> scriptContext) {
152+
throw new UnsupportedOperationException();
153+
}
154+
},
155+
bitsetFilterCache::getBitSetProducer,
156+
MapperMetrics.NOOP
157+
);
158+
159+
mapperServices.add(mapperService);
160+
}
161+
162+
compressedMapping = new CompressedXContent(MAPPING);
163+
}
164+
165+
@Benchmark
166+
public void mappingParsingBenchmark() {
167+
for (MapperService service : mapperServices) {
168+
service.merge("_doc", compressedMapping, MapperService.MergeReason.MAPPING_UPDATE);
169+
}
170+
}
171+
}

‎server/src/main/java/org/elasticsearch/index/IndexSortConfig.java‎

Lines changed: 100 additions & 61 deletions
Original file line numberDiff line numberDiff line change
@@ -27,7 +27,7 @@
2727
import org.elasticsearch.search.sort.SortOrder;
2828

2929
import java.util.ArrayList;
30-
import java.util.Arrays;
30+
import java.util.Collections;
3131
import java.util.EnumSet;
3232
import java.util.List;
3333
import java.util.Locale;
@@ -107,31 +107,68 @@ public final class IndexSortConfig {
107107
);
108108

109109
public static class IndexSortConfigDefaults {
110-
public static final FieldSortSpec[] TIME_SERIES_SORT, HOSTNAME_TIMESTAMP_BWC_SORT;
110+
public record SortDefault(List<String> fields, List<String> order, List<String> mode, List<String> missing) {
111+
public SortDefault {
112+
assert fields.size() == order.size();
113+
assert fields.size() == mode.size();
114+
assert fields.size() == missing.size();
115+
}
116+
}
111117

112-
private static final FieldSortSpec HOSTNAME_SPEC, MESSAGE_PATTERN_SPEC, TIMESTAMP_SPEC;
118+
public static final SortDefault NO_SORT, TIME_SERIES_SORT, TIMESTAMP_SORT, HOSTNAME_TIMESTAMP_SORT, HOSTNAME_TIMESTAMP_BWC_SORT,
119+
MESSAGE_PATTERN_TIMESTAMP_SORT, HOSTNAME_MESSAGE_PATTERN_TIMESTAMP_SORT;
113120

114121
static {
115-
TIMESTAMP_SPEC = new FieldSortSpec(DataStreamTimestampFieldMapper.DEFAULT_PATH);
116-
TIMESTAMP_SPEC.order = SortOrder.DESC;
117-
TIME_SERIES_SORT = new FieldSortSpec[] { new FieldSortSpec(TimeSeriesIdFieldMapper.NAME), TIMESTAMP_SPEC };
122+
NO_SORT = new SortDefault(Collections.emptyList(), Collections.emptyList(), Collections.emptyList(), Collections.emptyList());
118123

119-
HOSTNAME_SPEC = new FieldSortSpec(IndexMode.HOST_NAME);
120-
HOSTNAME_SPEC.order = SortOrder.ASC;
121-
HOSTNAME_SPEC.missingValue = "_last";
122-
HOSTNAME_SPEC.mode = MultiValueMode.MIN;
124+
TIME_SERIES_SORT = new SortDefault(
125+
List.of(TimeSeriesIdFieldMapper.NAME, DataStreamTimestampFieldMapper.DEFAULT_PATH),
126+
List.of("asc", "desc"),
127+
List.of("min", "max"),
128+
List.of("_last", "_last")
129+
);
123130

124-
MESSAGE_PATTERN_SPEC = new FieldSortSpec("message.template_id");
131+
TIMESTAMP_SORT = new SortDefault(
132+
List.of(DataStreamTimestampFieldMapper.DEFAULT_PATH),
133+
List.of("desc"),
134+
List.of("max"),
135+
List.of("_last")
136+
);
137+
138+
HOSTNAME_TIMESTAMP_SORT = new SortDefault(
139+
List.of(IndexMode.HOST_NAME, DataStreamTimestampFieldMapper.DEFAULT_PATH),
140+
List.of("asc", "desc"),
141+
List.of("min", "max"),
142+
List.of("_last", "_last")
143+
);
144+
145+
MESSAGE_PATTERN_TIMESTAMP_SORT = new SortDefault(
146+
List.of("message.template_id", DataStreamTimestampFieldMapper.DEFAULT_PATH),
147+
List.of("asc", "desc"),
148+
List.of("min", "max"),
149+
List.of("_last", "_last")
150+
);
151+
152+
HOSTNAME_MESSAGE_PATTERN_TIMESTAMP_SORT = new SortDefault(
153+
List.of(IndexMode.HOST_NAME, "message.template_id", DataStreamTimestampFieldMapper.DEFAULT_PATH),
154+
List.of("asc", "asc", "desc"),
155+
List.of("min", "min", "max"),
156+
List.of("_last", "_last", "_last")
157+
);
125158

126159
// Older indexes use ascending ordering for host name and timestamp.
127-
HOSTNAME_TIMESTAMP_BWC_SORT = new FieldSortSpec[] {
128-
new FieldSortSpec(IndexMode.HOST_NAME),
129-
new FieldSortSpec(DataStreamTimestampFieldMapper.DEFAULT_PATH) };
160+
HOSTNAME_TIMESTAMP_BWC_SORT = new SortDefault(
161+
List.of(IndexMode.HOST_NAME, DataStreamTimestampFieldMapper.DEFAULT_PATH),
162+
List.of("asc", "asc"),
163+
List.of("min", "min"),
164+
List.of("_last", "_last")
165+
);
166+
130167
}
131168

132-
public static FieldSortSpec[] getDefaultSortSpecs(Settings settings) {
169+
static SortDefault getSortDefault(Settings settings) {
133170
if (settings.isEmpty()) {
134-
return new FieldSortSpec[0];
171+
return NO_SORT;
135172
}
136173

137174
// Can't use IndexSettings.MODE.get(settings) here because the validation logic for IndexSettings.MODE uses the default value
@@ -152,71 +189,73 @@ public static FieldSortSpec[] getDefaultSortSpecs(Settings settings) {
152189
IndexVersions.UPGRADE_TO_LUCENE_10_0_0
153190
)) {
154191

155-
List<FieldSortSpec> sortSpecs = new ArrayList<>(3);
156-
if (IndexSettings.LOGSDB_SORT_ON_HOST_NAME.get(settings)) {
157-
sortSpecs.add(HOSTNAME_SPEC);
158-
}
159-
if (IndexSettings.LOGSDB_SORT_ON_MESSAGE_TEMPLATE.get(settings)) {
160-
sortSpecs.add(MESSAGE_PATTERN_SPEC);
192+
boolean sortOnHostName = settings.getAsBoolean(IndexSettings.LOGSDB_SORT_ON_HOST_NAME.getKey(), false);
193+
boolean sortOnMessageTemplate = settings.getAsBoolean(IndexSettings.LOGSDB_SORT_ON_MESSAGE_TEMPLATE.getKey(), false);
194+
if (sortOnHostName && sortOnMessageTemplate) {
195+
return HOSTNAME_MESSAGE_PATTERN_TIMESTAMP_SORT;
196+
} else if (sortOnHostName) {
197+
return HOSTNAME_TIMESTAMP_SORT;
198+
} else if (sortOnMessageTemplate) {
199+
return MESSAGE_PATTERN_TIMESTAMP_SORT;
200+
} else {
201+
return TIMESTAMP_SORT;
161202
}
162-
sortSpecs.add(TIMESTAMP_SPEC);
163-
164-
return sortSpecs.toArray(FieldSortSpec[]::new);
165203
} else {
166204
return HOSTNAME_TIMESTAMP_BWC_SORT;
167205
}
168206
}
169207

170-
return new FieldSortSpec[0];
171-
}
172-
173-
public static FieldSortSpec[] getSortSpecs(Settings settings) {
174-
if (INDEX_SORT_FIELD_SETTING.exists(settings) == false) {
175-
return IndexSortConfigDefaults.getDefaultSortSpecs(settings);
176-
}
177-
178-
List<String> fields = INDEX_SORT_FIELD_SETTING.get(settings);
179-
FieldSortSpec[] sortSpecs = fields.stream().map(FieldSortSpec::new).toArray(FieldSortSpec[]::new);
180-
181-
// Need to populate `order` because the default value of `mode` depends on it
182-
if (INDEX_SORT_ORDER_SETTING.exists(settings)) {
183-
List<SortOrder> orders = INDEX_SORT_ORDER_SETTING.get(settings);
184-
for (int i = 0; i < sortSpecs.length; i++) {
185-
sortSpecs[i].order = orders.get(i);
186-
}
187-
}
188-
189-
return sortSpecs;
208+
return NO_SORT;
190209
}
191210

192211
public static List<String> getDefaultSortFields(Settings settings) {
193-
return Arrays.stream(getDefaultSortSpecs(settings)).map(sortSpec -> sortSpec.field).toList();
212+
return getSortDefault(settings).fields();
194213
}
195214

196215
public static List<String> getDefaultSortOrder(Settings settings) {
197-
return Arrays.stream(getSortSpecs(settings))
198-
.map(sortSpec -> sortSpec.order != null ? sortSpec.order : SortOrder.ASC)
199-
.map(Enum::toString)
200-
.toList();
216+
if (settings.hasValue(INDEX_SORT_FIELD_SETTING.getKey()) == false) {
217+
return getSortDefault(settings).order();
218+
}
219+
220+
List<String> sortFields = settings.getAsList(INDEX_SORT_FIELD_SETTING.getKey());
221+
List<String> order = new ArrayList<>(sortFields.size());
222+
for (int i = 0; i < sortFields.size(); ++i) {
223+
order.add("asc");
224+
}
225+
return order;
201226
}
202227

203228
public static List<String> getDefaultSortMode(Settings settings) {
204-
return Arrays.stream(getSortSpecs(settings)).map(sortSpec -> {
205-
if (sortSpec.mode != null) {
206-
return sortSpec.mode;
207-
} else if (sortSpec.order == SortOrder.DESC) {
208-
return MultiValueMode.MAX;
229+
if (settings.hasValue(INDEX_SORT_FIELD_SETTING.getKey()) == false) {
230+
return getSortDefault(settings).mode();
231+
}
232+
233+
List<String> sortFields = settings.getAsList(INDEX_SORT_FIELD_SETTING.getKey());
234+
List<String> sortOrder = settings.getAsList(INDEX_SORT_ORDER_SETTING.getKey(), null);
235+
236+
List<String> mode = new ArrayList<>(sortFields.size());
237+
for (int i = 0; i < sortFields.size(); ++i) {
238+
if (sortOrder != null && sortOrder.get(i).equals(SortOrder.DESC.toString())) {
239+
mode.add("max");
209240
} else {
210-
return MultiValueMode.MIN;
241+
mode.add("min");
211242
}
212-
}).map(order -> order.toString().toLowerCase(Locale.ROOT)).toList();
243+
}
244+
return mode;
213245
}
214246

215247
public static List<String> getDefaultSortMissing(Settings settings) {
216-
// _last is the default per IndexFieldData.XFieldComparatorSource.Nested#sortMissingLast
217-
return Arrays.stream(getSortSpecs(settings))
218-
.map(sortSpec -> sortSpec.missingValue != null ? sortSpec.missingValue : "_last")
219-
.toList();
248+
if (settings.hasValue(INDEX_SORT_FIELD_SETTING.getKey()) == false) {
249+
return getSortDefault(settings).missing();
250+
}
251+
252+
List<String> sortFields = settings.getAsList(INDEX_SORT_FIELD_SETTING.getKey());
253+
List<String> missing = new ArrayList<>(sortFields.size());
254+
for (int i = 0; i < sortFields.size(); ++i) {
255+
// _last is the default per IndexFieldData.XFieldComparatorSource.Nested#sortMissingLast
256+
missing.add("_last");
257+
}
258+
return missing;
220259
}
221260
}
222261

0 commit comments

Comments
 (0)