Skip to content

Commit e83b779

Browse files
pabloemgmjehovich
pabloem
authored and
gmjehovich
committed
Increasing default limit for TS queries (elastic#134501)
* Increasing default limit for TS queries * fixup * matching TS +stats aggregations only * fixup * fixtest * comments
1 parent ca04a1e commit e83b779

File tree

18 files changed

+188
-49
lines changed

18 files changed

+188
-49
lines changed

‎benchmarks/src/main/java/org/elasticsearch/benchmark/_nightly/esql/QueryPlanningBenchmark.java‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -86,7 +86,9 @@ public void setup() {
8686
false,
8787
Map.of(),
8888
System.nanoTime(),
89-
false
89+
false,
90+
EsqlPlugin.QUERY_TIMESERIES_RESULT_TRUNCATION_DEFAULT_SIZE.getDefault(Settings.EMPTY),
91+
EsqlPlugin.QUERY_TIMESERIES_RESULT_TRUNCATION_DEFAULT_SIZE.get(Settings.EMPTY)
9092
);
9193

9294
var fields = 10_000;

‎benchmarks/src/main/java/org/elasticsearch/benchmark/compute/operator/EvalBenchmark.java‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -365,7 +365,9 @@ private static Configuration configuration() {
365365
false,
366366
Map.of(),
367367
0,
368-
false
368+
false,
369+
EsqlPlugin.QUERY_TIMESERIES_RESULT_TRUNCATION_MAX_SIZE.getDefault(Settings.EMPTY),
370+
EsqlPlugin.QUERY_TIMESERIES_RESULT_TRUNCATION_DEFAULT_SIZE.getDefault(Settings.EMPTY)
369371
);
370372
}
371373

‎server/src/main/java/org/elasticsearch/TransportVersions.java‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -325,6 +325,7 @@ static TransportVersion def(int id) {
325325
public static final TransportVersion ML_INFERENCE_ENDPOINT_CACHE = def(9_157_0_00);
326326
public static final TransportVersion INDEX_SOURCE = def(9_158_0_00);
327327
public static final TransportVersion MAX_HEAP_SIZE_PER_NODE_IN_CLUSTER_INFO = def(9_159_0_00);
328+
public static final TransportVersion TIMESERIES_DEFAULT_LIMIT = def(9_160_0_00);
328329

329330
/*
330331
* STOP! READ THIS FIRST! No, really,

‎x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/ConfigurationTestUtils.java‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -58,6 +58,8 @@ public static Configuration randomConfiguration(String query, Map<String, Map<St
5858
var clusterName = randomAlphaOfLengthBetween(3, 10);
5959
var truncation = randomNonNegativeInt();
6060
var defaultTruncation = randomNonNegativeInt();
61+
var tsTruncation = truncation + randomNonNegativeInt();
62+
var defaultTsTruncation = defaultTruncation + randomNonNegativeInt();
6163
boolean profile = randomBoolean();
6264

6365
return new Configuration(
@@ -72,7 +74,9 @@ public static Configuration randomConfiguration(String query, Map<String, Map<St
7274
profile,
7375
tables,
7476
System.nanoTime(),
75-
false
77+
false,
78+
tsTruncation,
79+
defaultTsTruncation
7680
);
7781
}
7882

‎x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/EsqlTestUtils.java‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -460,7 +460,9 @@ public static Configuration configuration(QueryPragmas pragmas, String query) {
460460
false,
461461
TABLES,
462462
System.nanoTime(),
463-
false
463+
false,
464+
EsqlPlugin.QUERY_TIMESERIES_RESULT_TRUNCATION_MAX_SIZE.getDefault(Settings.EMPTY),
465+
EsqlPlugin.QUERY_TIMESERIES_RESULT_TRUNCATION_DEFAULT_SIZE.getDefault(Settings.EMPTY)
464466
);
465467
}
466468

‎x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java‎

Lines changed: 14 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -114,6 +114,7 @@
114114
import org.elasticsearch.xpack.esql.plan.logical.MvExpand;
115115
import org.elasticsearch.xpack.esql.plan.logical.Project;
116116
import org.elasticsearch.xpack.esql.plan.logical.Rename;
117+
import org.elasticsearch.xpack.esql.plan.logical.TimeSeriesAggregate;
117118
import org.elasticsearch.xpack.esql.plan.logical.UnresolvedRelation;
118119
import org.elasticsearch.xpack.esql.plan.logical.fuse.Fuse;
119120
import org.elasticsearch.xpack.esql.plan.logical.fuse.FuseScoreEval;
@@ -1372,15 +1373,22 @@ private static class AddImplicitLimit extends ParameterizedRule<LogicalPlan, Log
13721373
@Override
13731374
public LogicalPlan apply(LogicalPlan logicalPlan, AnalyzerContext context) {
13741375
List<LogicalPlan> limits = logicalPlan.collectFirstChildren(Limit.class::isInstance);
1376+
// We check whether the query contains a TimeSeriesAggregate to determine if we should apply
1377+
// the default limit for TS queries or for non-TS queries.
1378+
boolean isTsAggregate = logicalPlan.collectFirstChildren(lp -> lp instanceof TimeSeriesAggregate)
1379+
.stream()
1380+
.toList()
1381+
.isEmpty() == false;
13751382
int limit;
13761383
if (limits.isEmpty()) {
1377-
HeaderWarning.addWarning(
1378-
"No limit defined, adding default limit of [{}]",
1379-
context.configuration().resultTruncationDefaultSize()
1380-
);
1381-
limit = context.configuration().resultTruncationDefaultSize(); // user provided no limit: cap to a default
1384+
limit = context.configuration().resultTruncationDefaultSize(isTsAggregate); // user provided no limit: cap to a
1385+
// default
1386+
if (isTsAggregate == false) {
1387+
HeaderWarning.addWarning("No limit defined, adding default limit of [{}]", limit);
1388+
}
13821389
} else {
1383-
limit = context.configuration().resultTruncationMaxSize(); // user provided a limit: cap result entries to the max
1390+
limit = context.configuration().resultTruncationMaxSize(isTsAggregate); // user provided a limit: cap result
1391+
// entries to the max
13841392
}
13851393
var source = logicalPlan.source();
13861394
return new Limit(source, new Literal(source, limit, DataType.INTEGER), logicalPlan);

‎x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/EsqlPlugin.java‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -113,6 +113,24 @@ public class EsqlPlugin extends Plugin implements ActionPlugin, ExtensiblePlugin
113113
Setting.Property.Dynamic
114114
);
115115

116+
public static final Setting<Integer> QUERY_TIMESERIES_RESULT_TRUNCATION_DEFAULT_SIZE = Setting.intSetting(
117+
"esql.query.timeseries_result_truncation_default_size",
118+
1_000_000,
119+
1,
120+
10_000_000,
121+
Setting.Property.NodeScope,
122+
Setting.Property.Dynamic
123+
);
124+
125+
public static final Setting<Integer> QUERY_TIMESERIES_RESULT_TRUNCATION_MAX_SIZE = Setting.intSetting(
126+
"esql.query.timeseries_result_truncation_max_size",
127+
10_000_000,
128+
1,
129+
1_000_000_000,
130+
Setting.Property.NodeScope,
131+
Setting.Property.Dynamic
132+
);
133+
116134
public static final Setting<Boolean> QUERY_ALLOW_PARTIAL_RESULTS = Setting.boolSetting(
117135
"esql.query.allow_partial_results",
118136
true,
@@ -250,6 +268,8 @@ public List<Setting<?>> getSettings() {
250268
return List.of(
251269
QUERY_RESULT_TRUNCATION_DEFAULT_SIZE,
252270
QUERY_RESULT_TRUNCATION_MAX_SIZE,
271+
QUERY_TIMESERIES_RESULT_TRUNCATION_DEFAULT_SIZE,
272+
QUERY_TIMESERIES_RESULT_TRUNCATION_MAX_SIZE,
253273
QUERY_ALLOW_PARTIAL_RESULTS,
254274
ESQL_QUERYLOG_THRESHOLD_TRACE_SETTING,
255275
ESQL_QUERYLOG_THRESHOLD_DEBUG_SETTING,

‎x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/TransportEsqlQueryAction.java‎

Lines changed: 27 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -86,6 +86,10 @@ public class TransportEsqlQueryAction extends HandledTransportAction<EsqlQueryRe
8686
private final UsageService usageService;
8787
private final TransportActionServices services;
8888
private volatile boolean defaultAllowPartialResults;
89+
private volatile int resultTruncationMaxSize;
90+
private volatile int resultTruncationDefaultSize;
91+
private volatile int timeseriesResultTruncationMaxSize;
92+
private volatile int timeseriesResultTruncationDefaultSize;
8993

9094
@Inject
9195
@SuppressWarnings("this-escape")
@@ -181,7 +185,24 @@ public TransportEsqlQueryAction(
181185
defaultAllowPartialResults = EsqlPlugin.QUERY_ALLOW_PARTIAL_RESULTS.get(clusterService.getSettings());
182186
clusterService.getClusterSettings()
183187
.addSettingsUpdateConsumer(EsqlPlugin.QUERY_ALLOW_PARTIAL_RESULTS, v -> defaultAllowPartialResults = v);
184-
188+
resultTruncationMaxSize = EsqlPlugin.QUERY_RESULT_TRUNCATION_MAX_SIZE.get(clusterService.getSettings());
189+
resultTruncationDefaultSize = EsqlPlugin.QUERY_RESULT_TRUNCATION_DEFAULT_SIZE.get(clusterService.getSettings());
190+
timeseriesResultTruncationMaxSize = EsqlPlugin.QUERY_TIMESERIES_RESULT_TRUNCATION_MAX_SIZE.get(clusterService.getSettings());
191+
timeseriesResultTruncationDefaultSize = EsqlPlugin.QUERY_TIMESERIES_RESULT_TRUNCATION_DEFAULT_SIZE.get(
192+
clusterService.getSettings()
193+
);
194+
clusterService.getClusterSettings().addSettingsUpdateConsumer(EsqlPlugin.QUERY_RESULT_TRUNCATION_MAX_SIZE, v -> {
195+
resultTruncationMaxSize = v;
196+
});
197+
clusterService.getClusterSettings().addSettingsUpdateConsumer(EsqlPlugin.QUERY_RESULT_TRUNCATION_DEFAULT_SIZE, v -> {
198+
resultTruncationDefaultSize = v;
199+
});
200+
clusterService.getClusterSettings().addSettingsUpdateConsumer(EsqlPlugin.QUERY_TIMESERIES_RESULT_TRUNCATION_MAX_SIZE, v -> {
201+
timeseriesResultTruncationMaxSize = v;
202+
});
203+
clusterService.getClusterSettings().addSettingsUpdateConsumer(EsqlPlugin.QUERY_TIMESERIES_RESULT_TRUNCATION_DEFAULT_SIZE, v -> {
204+
timeseriesResultTruncationDefaultSize = v;
205+
});
185206
}
186207

187208
@Override
@@ -229,13 +250,15 @@ private void innerExecute(Task task, EsqlQueryRequest request, ActionListener<Es
229250
null,
230251
clusterService.getClusterName().value(),
231252
request.pragmas(),
232-
clusterService.getClusterSettings().get(EsqlPlugin.QUERY_RESULT_TRUNCATION_MAX_SIZE),
233-
clusterService.getClusterSettings().get(EsqlPlugin.QUERY_RESULT_TRUNCATION_DEFAULT_SIZE),
253+
resultTruncationMaxSize,
254+
resultTruncationDefaultSize,
234255
request.query(),
235256
request.profile(),
236257
request.tables(),
237258
System.nanoTime(),
238-
request.allowPartialResults()
259+
request.allowPartialResults(),
260+
timeseriesResultTruncationMaxSize,
261+
timeseriesResultTruncationDefaultSize
239262
);
240263
String sessionId = sessionID(task);
241264
// async-query uses EsqlQueryTask, so pull the EsqlExecutionInfo out of the task

‎x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/Configuration.java‎

Lines changed: 60 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -44,8 +44,10 @@ public class Configuration implements Writeable {
4444

4545
private final QueryPragmas pragmas;
4646

47-
private final int resultTruncationMaxSize;
48-
private final int resultTruncationDefaultSize;
47+
private final int resultTruncationMaxSizeRegular;
48+
private final int resultTruncationDefaultSizeRegular;
49+
private final int resultTruncationMaxSizeTimeseries;
50+
private final int resultTruncationDefaultSizeTimeseries;
4951

5052
private final Locale locale;
5153

@@ -63,22 +65,26 @@ public Configuration(
6365
String username,
6466
String clusterName,
6567
QueryPragmas pragmas,
66-
int resultTruncationMaxSize,
67-
int resultTruncationDefaultSize,
68+
int resultTruncationMaxSizeRegular,
69+
int resultTruncationDefaultSizeRegular,
6870
String query,
6971
boolean profile,
7072
Map<String, Map<String, Column>> tables,
7173
long queryStartTimeNanos,
72-
boolean allowPartialResults
74+
boolean allowPartialResults,
75+
int resultTruncationMaxSizeTimeseries,
76+
int resultTruncationDefaultSizeTimeseries
7377
) {
7478
this.zoneId = zi.normalized();
7579
this.now = ZonedDateTime.now(Clock.tick(Clock.system(zoneId), Duration.ofNanos(1)));
7680
this.username = username;
7781
this.clusterName = clusterName;
7882
this.locale = locale;
7983
this.pragmas = pragmas;
80-
this.resultTruncationMaxSize = resultTruncationMaxSize;
81-
this.resultTruncationDefaultSize = resultTruncationDefaultSize;
84+
this.resultTruncationMaxSizeRegular = resultTruncationMaxSizeRegular;
85+
this.resultTruncationDefaultSizeRegular = resultTruncationDefaultSizeRegular;
86+
this.resultTruncationMaxSizeTimeseries = resultTruncationMaxSizeTimeseries;
87+
this.resultTruncationDefaultSizeTimeseries = resultTruncationDefaultSizeTimeseries;
8288
this.query = query;
8389
this.profile = profile;
8490
this.tables = tables;
@@ -94,8 +100,8 @@ public Configuration(BlockStreamInput in) throws IOException {
94100
this.clusterName = in.readOptionalString();
95101
locale = Locale.forLanguageTag(in.readString());
96102
this.pragmas = new QueryPragmas(in);
97-
this.resultTruncationMaxSize = in.readVInt();
98-
this.resultTruncationDefaultSize = in.readVInt();
103+
this.resultTruncationMaxSizeRegular = in.readVInt();
104+
this.resultTruncationDefaultSizeRegular = in.readVInt();
99105
this.query = readQuery(in);
100106
if (in.getTransportVersion().onOrAfter(TransportVersions.V_8_12_0)) {
101107
this.profile = in.readBoolean();
@@ -118,6 +124,13 @@ public Configuration(BlockStreamInput in) throws IOException {
118124
} else {
119125
this.allowPartialResults = false;
120126
}
127+
if (in.getTransportVersion().onOrAfter(TransportVersions.TIMESERIES_DEFAULT_LIMIT)) {
128+
this.resultTruncationMaxSizeTimeseries = in.readVInt();
129+
this.resultTruncationDefaultSizeTimeseries = in.readVInt();
130+
} else {
131+
this.resultTruncationMaxSizeTimeseries = this.resultTruncationMaxSizeRegular;
132+
this.resultTruncationDefaultSizeTimeseries = this.resultTruncationDefaultSizeRegular;
133+
}
121134
}
122135

123136
@Override
@@ -130,8 +143,8 @@ public void writeTo(StreamOutput out) throws IOException {
130143
out.writeOptionalString(clusterName); // TODO this one is never null so maybe not optional
131144
out.writeString(locale.toLanguageTag());
132145
pragmas.writeTo(out);
133-
out.writeVInt(resultTruncationMaxSize);
134-
out.writeVInt(resultTruncationDefaultSize);
146+
out.writeVInt(resultTruncationMaxSizeRegular);
147+
out.writeVInt(resultTruncationDefaultSizeRegular);
135148
writeQuery(out, query);
136149
if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_12_0)) {
137150
out.writeBoolean(profile);
@@ -146,6 +159,10 @@ public void writeTo(StreamOutput out) throws IOException {
146159
|| out.getTransportVersion().isPatchFrom(TransportVersions.ESQL_SUPPORT_PARTIAL_RESULTS_BACKPORT_8_19)) {
147160
out.writeBoolean(allowPartialResults);
148161
}
162+
if (out.getTransportVersion().onOrAfter(TransportVersions.TIMESERIES_DEFAULT_LIMIT)) {
163+
out.writeVInt(resultTruncationMaxSizeTimeseries);
164+
out.writeVInt(resultTruncationDefaultSizeTimeseries);
165+
}
149166
}
150167

151168
public ZoneId zoneId() {
@@ -168,12 +185,18 @@ public QueryPragmas pragmas() {
168185
return pragmas;
169186
}
170187

171-
public int resultTruncationMaxSize() {
172-
return resultTruncationMaxSize;
188+
public int resultTruncationMaxSize(boolean isTimeseries) {
189+
if (isTimeseries) {
190+
return resultTruncationMaxSizeTimeseries;
191+
}
192+
return resultTruncationMaxSizeRegular;
173193
}
174194

175-
public int resultTruncationDefaultSize() {
176-
return resultTruncationDefaultSize;
195+
public int resultTruncationDefaultSize(boolean isTimeseries) {
196+
if (isTimeseries) {
197+
return resultTruncationDefaultSizeTimeseries;
198+
}
199+
return resultTruncationDefaultSizeRegular;
177200
}
178201

179202
public Locale locale() {
@@ -220,13 +243,15 @@ public Configuration withoutTables() {
220243
username,
221244
clusterName,
222245
pragmas,
223-
resultTruncationMaxSize,
224-
resultTruncationDefaultSize,
246+
resultTruncationMaxSizeRegular,
247+
resultTruncationDefaultSizeRegular,
225248
query,
226249
profile,
227250
Map.of(),
228251
queryStartTimeNanos,
229-
allowPartialResults
252+
allowPartialResults,
253+
resultTruncationMaxSizeTimeseries,
254+
resultTruncationDefaultSizeTimeseries
230255
);
231256
}
232257

@@ -277,8 +302,8 @@ public boolean equals(Object o) {
277302
&& Objects.equals(now, that.now)
278303
&& Objects.equals(username, that.username)
279304
&& Objects.equals(clusterName, that.clusterName)
280-
&& resultTruncationMaxSize == that.resultTruncationMaxSize
281-
&& resultTruncationDefaultSize == that.resultTruncationDefaultSize
305+
&& resultTruncationMaxSizeRegular == that.resultTruncationMaxSizeRegular
306+
&& resultTruncationDefaultSizeRegular == that.resultTruncationDefaultSizeRegular
282307
&& Objects.equals(pragmas, that.pragmas)
283308
&& Objects.equals(locale, that.locale)
284309
&& Objects.equals(that.query, query)
@@ -295,13 +320,15 @@ public int hashCode() {
295320
username,
296321
clusterName,
297322
pragmas,
298-
resultTruncationMaxSize,
299-
resultTruncationDefaultSize,
323+
resultTruncationMaxSizeRegular,
324+
resultTruncationDefaultSizeRegular,
300325
locale,
301326
query,
302327
profile,
303328
tables,
304-
allowPartialResults
329+
allowPartialResults,
330+
resultTruncationMaxSizeTimeseries,
331+
resultTruncationDefaultSizeTimeseries
305332
);
306333
}
307334

@@ -311,9 +338,17 @@ public String toString() {
311338
+ "pragmas="
312339
+ pragmas
313340
+ ", resultTruncationMaxSize="
314-
+ resultTruncationMaxSize
341+
+ "[regular="
342+
+ resultTruncationMaxSize(false)
343+
+ ",timeseries="
344+
+ resultTruncationMaxSize(true)
345+
+ "]"
315346
+ ", resultTruncationDefaultSize="
316-
+ resultTruncationDefaultSize
347+
+ "[regular="
348+
+ resultTruncationDefaultSize(false)
349+
+ ",timeseries="
350+
+ resultTruncationDefaultSize(true)
351+
+ "]"
317352
+ ", locale="
318353
+ locale
319354
+ ", query='"

0 commit comments

Comments
 (0)