Skip to content

Commit d0d39dc

Browse files
alex-spiesalex-spies
authored
ESQL: Propagate min transport version into query planning (elastic#136479)
1 parent 5f0f99a commit d0d39dc

File tree

50 files changed

+669
-330
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

50 files changed

+669
-330
lines changed

‎benchmarks/src/main/java/org/elasticsearch/benchmark/_nightly/esql/QueryPlanningBenchmark.java‎

Lines changed: 12 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -9,14 +9,12 @@
99

1010
package org.elasticsearch.benchmark._nightly.esql;
1111

12+
import org.elasticsearch.TransportVersion;
1213
import org.elasticsearch.common.logging.LogConfigurator;
1314
import org.elasticsearch.common.settings.Settings;
1415
import org.elasticsearch.index.IndexMode;
1516
import org.elasticsearch.license.XPackLicenseState;
16-
import org.elasticsearch.xpack.esql.analysis.Analyzer;
17-
import org.elasticsearch.xpack.esql.analysis.AnalyzerContext;
18-
import org.elasticsearch.xpack.esql.analysis.EnrichResolution;
19-
import org.elasticsearch.xpack.esql.analysis.Verifier;
17+
import org.elasticsearch.xpack.esql.analysis.*;
2018
import org.elasticsearch.xpack.esql.core.expression.FoldContext;
2119
import org.elasticsearch.xpack.esql.core.type.EsField;
2220
import org.elasticsearch.xpack.esql.core.util.DateUtils;
@@ -29,7 +27,6 @@
2927
import org.elasticsearch.xpack.esql.parser.EsqlParser;
3028
import org.elasticsearch.xpack.esql.parser.QueryParams;
3129
import org.elasticsearch.xpack.esql.plan.logical.LogicalPlan;
32-
import org.elasticsearch.xpack.esql.plugin.EsqlPlugin;
3330
import org.elasticsearch.xpack.esql.plugin.QueryPragmas;
3431
import org.elasticsearch.xpack.esql.session.Configuration;
3532
import org.elasticsearch.xpack.esql.telemetry.Metrics;
@@ -80,15 +77,15 @@ public void setup() {
8077
null,
8178
null,
8279
new QueryPragmas(Settings.EMPTY),
83-
EsqlPlugin.QUERY_RESULT_TRUNCATION_MAX_SIZE.getDefault(Settings.EMPTY),
84-
EsqlPlugin.QUERY_RESULT_TRUNCATION_DEFAULT_SIZE.getDefault(Settings.EMPTY),
80+
AnalyzerSettings.QUERY_RESULT_TRUNCATION_MAX_SIZE.getDefault(Settings.EMPTY),
81+
AnalyzerSettings.QUERY_RESULT_TRUNCATION_DEFAULT_SIZE.getDefault(Settings.EMPTY),
8582
"",
8683
false,
8784
Map.of(),
8885
System.nanoTime(),
8986
false,
90-
EsqlPlugin.QUERY_TIMESERIES_RESULT_TRUNCATION_DEFAULT_SIZE.getDefault(Settings.EMPTY),
91-
EsqlPlugin.QUERY_TIMESERIES_RESULT_TRUNCATION_DEFAULT_SIZE.get(Settings.EMPTY)
87+
AnalyzerSettings.QUERY_TIMESERIES_RESULT_TRUNCATION_DEFAULT_SIZE.getDefault(Settings.EMPTY),
88+
AnalyzerSettings.QUERY_TIMESERIES_RESULT_TRUNCATION_DEFAULT_SIZE.get(Settings.EMPTY)
9289
);
9390

9491
var fields = 10_000;
@@ -102,6 +99,9 @@ public void setup() {
10299

103100
var functionRegistry = new EsqlFunctionRegistry();
104101

102+
// Assume all nodes are on the current version for the benchmark.
103+
TransportVersion minimumVersion = TransportVersion.current();
104+
105105
telemetry = new PlanTelemetry(functionRegistry);
106106
defaultParser = new EsqlParser();
107107
manyFieldsAnalyzer = new Analyzer(
@@ -111,11 +111,12 @@ public void setup() {
111111
IndexResolution.valid(esIndex),
112112
Map.of(),
113113
new EnrichResolution(),
114-
InferenceResolution.EMPTY
114+
InferenceResolution.EMPTY,
115+
minimumVersion
115116
),
116117
new Verifier(new Metrics(functionRegistry), new XPackLicenseState(() -> 0L))
117118
);
118-
defaultOptimizer = new LogicalPlanOptimizer(new LogicalOptimizerContext(config, FoldContext.small()));
119+
defaultOptimizer = new LogicalPlanOptimizer(new LogicalOptimizerContext(config, FoldContext.small(), minimumVersion));
119120
}
120121

121122
private LogicalPlan plan(EsqlParser parser, Analyzer analyzer, LogicalPlanOptimizer optimizer, String query) {

‎benchmarks/src/main/java/org/elasticsearch/benchmark/compute/operator/EvalBenchmark.java‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,7 @@
3333
import org.elasticsearch.core.TimeValue;
3434
import org.elasticsearch.logging.LogManager;
3535
import org.elasticsearch.logging.Logger;
36+
import org.elasticsearch.xpack.esql.analysis.AnalyzerSettings;
3637
import org.elasticsearch.xpack.esql.core.expression.Expression;
3738
import org.elasticsearch.xpack.esql.core.expression.FieldAttribute;
3839
import org.elasticsearch.xpack.esql.core.expression.FoldContext;
@@ -55,7 +56,6 @@
5556
import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.Equals;
5657
import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.LessThan;
5758
import org.elasticsearch.xpack.esql.planner.Layout;
58-
import org.elasticsearch.xpack.esql.plugin.EsqlPlugin;
5959
import org.elasticsearch.xpack.esql.session.Configuration;
6060
import org.openjdk.jmh.annotations.Benchmark;
6161
import org.openjdk.jmh.annotations.BenchmarkMode;
@@ -359,15 +359,15 @@ private static Configuration configuration() {
359359
null,
360360
null,
361361
null,
362-
EsqlPlugin.QUERY_RESULT_TRUNCATION_MAX_SIZE.get(Settings.EMPTY),
363-
EsqlPlugin.QUERY_RESULT_TRUNCATION_DEFAULT_SIZE.get(Settings.EMPTY),
362+
AnalyzerSettings.QUERY_RESULT_TRUNCATION_MAX_SIZE.get(Settings.EMPTY),
363+
AnalyzerSettings.QUERY_RESULT_TRUNCATION_DEFAULT_SIZE.get(Settings.EMPTY),
364364
null,
365365
false,
366366
Map.of(),
367367
0,
368368
false,
369-
EsqlPlugin.QUERY_TIMESERIES_RESULT_TRUNCATION_MAX_SIZE.getDefault(Settings.EMPTY),
370-
EsqlPlugin.QUERY_TIMESERIES_RESULT_TRUNCATION_DEFAULT_SIZE.getDefault(Settings.EMPTY)
369+
AnalyzerSettings.QUERY_TIMESERIES_RESULT_TRUNCATION_MAX_SIZE.getDefault(Settings.EMPTY),
370+
AnalyzerSettings.QUERY_TIMESERIES_RESULT_TRUNCATION_DEFAULT_SIZE.getDefault(Settings.EMPTY)
371371
);
372372
}
373373

‎x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/FieldAttribute.java‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -48,7 +48,8 @@ public record FieldName(String string) {};
4848
FieldAttribute::readFrom
4949
);
5050

51-
private static final TransportVersion ESQL_FIELD_ATTRIBUTE_DROP_TYPE = TransportVersion.fromName("esql_field_attribute_drop_type");
51+
// Only public for testing
52+
public static final TransportVersion ESQL_FIELD_ATTRIBUTE_DROP_TYPE = TransportVersion.fromName("esql_field_attribute_drop_type");
5253

5354
private final String parentName;
5455
private final EsField field;

‎x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/EsqlTestUtils.java‎

Lines changed: 53 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@
1212
import org.apache.lucene.sandbox.document.HalfFloatPoint;
1313
import org.apache.lucene.util.BytesRef;
1414
import org.elasticsearch.ExceptionsHelper;
15+
import org.elasticsearch.TransportVersion;
1516
import org.elasticsearch.client.internal.Client;
1617
import org.elasticsearch.cluster.ClusterName;
1718
import org.elasticsearch.cluster.RemoteException;
@@ -54,12 +55,15 @@
5455
import org.elasticsearch.search.aggregations.bucket.geogrid.GeoTileUtils;
5556
import org.elasticsearch.tasks.TaskCancelledException;
5657
import org.elasticsearch.test.ESTestCase;
58+
import org.elasticsearch.test.TransportVersionUtils;
5759
import org.elasticsearch.threadpool.ThreadPool;
5860
import org.elasticsearch.transport.RemoteTransportException;
5961
import org.elasticsearch.transport.TransportService;
6062
import org.elasticsearch.xcontent.XContentType;
6163
import org.elasticsearch.xcontent.json.JsonXContent;
6264
import org.elasticsearch.xpack.esql.action.EsqlQueryResponse;
65+
import org.elasticsearch.xpack.esql.analysis.AnalyzerContext;
66+
import org.elasticsearch.xpack.esql.analysis.AnalyzerSettings;
6367
import org.elasticsearch.xpack.esql.analysis.EnrichResolution;
6468
import org.elasticsearch.xpack.esql.analysis.Verifier;
6569
import org.elasticsearch.xpack.esql.core.expression.Attribute;
@@ -90,6 +94,7 @@
9094
import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.LessThanOrEqual;
9195
import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.NotEquals;
9296
import org.elasticsearch.xpack.esql.index.EsIndex;
97+
import org.elasticsearch.xpack.esql.index.IndexResolution;
9398
import org.elasticsearch.xpack.esql.inference.InferenceResolution;
9499
import org.elasticsearch.xpack.esql.inference.InferenceService;
95100
import org.elasticsearch.xpack.esql.optimizer.LogicalOptimizerContext;
@@ -103,8 +108,6 @@
103108
import org.elasticsearch.xpack.esql.plan.logical.local.LocalSupplier;
104109
import org.elasticsearch.xpack.esql.planner.PlannerSettings;
105110
import org.elasticsearch.xpack.esql.planner.PlannerUtils;
106-
import org.elasticsearch.xpack.esql.plugin.EsqlPlugin;
107-
import org.elasticsearch.xpack.esql.plugin.EsqlQueryClusterSettings;
108111
import org.elasticsearch.xpack.esql.plugin.QueryPragmas;
109112
import org.elasticsearch.xpack.esql.plugin.TransportActionServices;
110113
import org.elasticsearch.xpack.esql.session.Configuration;
@@ -430,8 +433,45 @@ public Object max(FieldName field) {
430433

431434
public static final Configuration TEST_CFG = configuration(new QueryPragmas(Settings.EMPTY));
432435

436+
public static TransportVersion randomMinimumVersion() {
437+
return TransportVersionUtils.randomCompatibleVersion(ESTestCase.random());
438+
}
439+
440+
// TODO: make this even simpler, remove the enrichResolution for tests that do not require it (most tests)
441+
public static AnalyzerContext testAnalyzerContext(
442+
Configuration configuration,
443+
EsqlFunctionRegistry functionRegistry,
444+
IndexResolution indexResolution,
445+
EnrichResolution enrichResolution,
446+
InferenceResolution inferenceResolution
447+
) {
448+
return testAnalyzerContext(configuration, functionRegistry, indexResolution, Map.of(), enrichResolution, inferenceResolution);
449+
}
450+
451+
/**
452+
* Analyzer context for a random (but compatible) minimum transport version.
453+
*/
454+
public static AnalyzerContext testAnalyzerContext(
455+
Configuration configuration,
456+
EsqlFunctionRegistry functionRegistry,
457+
IndexResolution indexResolution,
458+
Map<String, IndexResolution> lookupResolution,
459+
EnrichResolution enrichResolution,
460+
InferenceResolution inferenceResolution
461+
) {
462+
return new AnalyzerContext(
463+
configuration,
464+
functionRegistry,
465+
indexResolution,
466+
lookupResolution,
467+
enrichResolution,
468+
inferenceResolution,
469+
randomMinimumVersion()
470+
);
471+
}
472+
433473
public static LogicalOptimizerContext unboundLogicalOptimizerContext() {
434-
return new LogicalOptimizerContext(EsqlTestUtils.TEST_CFG, FoldContext.small());
474+
return new LogicalOptimizerContext(EsqlTestUtils.TEST_CFG, FoldContext.small(), randomMinimumVersion());
435475
}
436476

437477
public static final Verifier TEST_VERIFIER = new Verifier(new Metrics(new EsqlFunctionRegistry()), new XPackLicenseState(() -> 0L));
@@ -483,15 +523,15 @@ public static Configuration configuration(QueryPragmas pragmas, String query) {
483523
null,
484524
null,
485525
pragmas,
486-
EsqlPlugin.QUERY_RESULT_TRUNCATION_MAX_SIZE.getDefault(Settings.EMPTY),
487-
EsqlPlugin.QUERY_RESULT_TRUNCATION_DEFAULT_SIZE.getDefault(Settings.EMPTY),
526+
AnalyzerSettings.QUERY_RESULT_TRUNCATION_MAX_SIZE.getDefault(Settings.EMPTY),
527+
AnalyzerSettings.QUERY_RESULT_TRUNCATION_DEFAULT_SIZE.getDefault(Settings.EMPTY),
488528
query,
489529
false,
490530
TABLES,
491531
System.nanoTime(),
492532
false,
493-
EsqlPlugin.QUERY_TIMESERIES_RESULT_TRUNCATION_MAX_SIZE.getDefault(Settings.EMPTY),
494-
EsqlPlugin.QUERY_TIMESERIES_RESULT_TRUNCATION_DEFAULT_SIZE.getDefault(Settings.EMPTY)
533+
AnalyzerSettings.QUERY_TIMESERIES_RESULT_TRUNCATION_MAX_SIZE.getDefault(Settings.EMPTY),
534+
AnalyzerSettings.QUERY_TIMESERIES_RESULT_TRUNCATION_DEFAULT_SIZE.getDefault(Settings.EMPTY)
495535
);
496536
}
497537

@@ -503,12 +543,12 @@ public static Configuration configuration(String query) {
503543
return configuration(new QueryPragmas(Settings.EMPTY), query);
504544
}
505545

506-
public static EsqlQueryClusterSettings queryClusterSettings() {
507-
return new EsqlQueryClusterSettings(
508-
EsqlPlugin.QUERY_RESULT_TRUNCATION_MAX_SIZE.getDefault(Settings.EMPTY),
509-
EsqlPlugin.QUERY_RESULT_TRUNCATION_DEFAULT_SIZE.getDefault(Settings.EMPTY),
510-
EsqlPlugin.QUERY_TIMESERIES_RESULT_TRUNCATION_MAX_SIZE.getDefault(Settings.EMPTY),
511-
EsqlPlugin.QUERY_TIMESERIES_RESULT_TRUNCATION_DEFAULT_SIZE.getDefault(Settings.EMPTY)
546+
public static AnalyzerSettings queryClusterSettings() {
547+
return new AnalyzerSettings(
548+
AnalyzerSettings.QUERY_RESULT_TRUNCATION_MAX_SIZE.getDefault(Settings.EMPTY),
549+
AnalyzerSettings.QUERY_RESULT_TRUNCATION_DEFAULT_SIZE.getDefault(Settings.EMPTY),
550+
AnalyzerSettings.QUERY_TIMESERIES_RESULT_TRUNCATION_MAX_SIZE.getDefault(Settings.EMPTY),
551+
AnalyzerSettings.QUERY_TIMESERIES_RESULT_TRUNCATION_DEFAULT_SIZE.getDefault(Settings.EMPTY)
512552
);
513553
}
514554

‎x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/EsqlActionIT.java‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -51,9 +51,9 @@
5151
import org.elasticsearch.xcontent.json.JsonXContent;
5252
import org.elasticsearch.xpack.core.esql.action.ColumnInfo;
5353
import org.elasticsearch.xpack.esql.VerificationException;
54+
import org.elasticsearch.xpack.esql.analysis.AnalyzerSettings;
5455
import org.elasticsearch.xpack.esql.core.type.DataType;
5556
import org.elasticsearch.xpack.esql.parser.ParsingException;
56-
import org.elasticsearch.xpack.esql.plugin.EsqlPlugin;
5757
import org.elasticsearch.xpack.esql.plugin.QueryPragmas;
5858
import org.junit.Before;
5959

@@ -1905,7 +1905,7 @@ private void createAndPopulateIndex(String indexName, Settings additionalSetting
19051905
public void testDefaultTruncationSizeSetting() {
19061906
ClusterAdminClient client = admin().cluster();
19071907

1908-
Settings settings = Settings.builder().put(EsqlPlugin.QUERY_RESULT_TRUNCATION_DEFAULT_SIZE.getKey(), 1).build();
1908+
Settings settings = Settings.builder().put(AnalyzerSettings.QUERY_RESULT_TRUNCATION_DEFAULT_SIZE.getKey(), 1).build();
19091909

19101910
ClusterUpdateSettingsRequest settingsRequest = new ClusterUpdateSettingsRequest(TEST_REQUEST_TIMEOUT, TEST_REQUEST_TIMEOUT)
19111911
.persistentSettings(settings);
@@ -1915,14 +1915,14 @@ public void testDefaultTruncationSizeSetting() {
19151915
logger.info(results);
19161916
assertEquals(1, getValuesList(results).size());
19171917
} finally {
1918-
clearPersistentSettings(EsqlPlugin.QUERY_RESULT_TRUNCATION_DEFAULT_SIZE);
1918+
clearPersistentSettings(AnalyzerSettings.QUERY_RESULT_TRUNCATION_DEFAULT_SIZE);
19191919
}
19201920
}
19211921

19221922
public void testMaxTruncationSizeSetting() {
19231923
ClusterAdminClient client = admin().cluster();
19241924

1925-
Settings settings = Settings.builder().put(EsqlPlugin.QUERY_RESULT_TRUNCATION_MAX_SIZE.getKey(), 10).build();
1925+
Settings settings = Settings.builder().put(AnalyzerSettings.QUERY_RESULT_TRUNCATION_MAX_SIZE.getKey(), 10).build();
19261926

19271927
ClusterUpdateSettingsRequest settingsRequest = new ClusterUpdateSettingsRequest(TEST_REQUEST_TIMEOUT, TEST_REQUEST_TIMEOUT)
19281928
.persistentSettings(settings);
@@ -1932,7 +1932,7 @@ public void testMaxTruncationSizeSetting() {
19321932
logger.info(results);
19331933
assertEquals(10, getValuesList(results).size());
19341934
} finally {
1935-
clearPersistentSettings(EsqlPlugin.QUERY_RESULT_TRUNCATION_MAX_SIZE);
1935+
clearPersistentSettings(AnalyzerSettings.QUERY_RESULT_TRUNCATION_MAX_SIZE);
19361936
}
19371937
}
19381938

‎x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/AnalyzerContext.java‎

Lines changed: 31 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -7,10 +7,12 @@
77

88
package org.elasticsearch.xpack.esql.analysis;
99

10+
import org.elasticsearch.TransportVersion;
1011
import org.elasticsearch.xpack.esql.expression.function.EsqlFunctionRegistry;
1112
import org.elasticsearch.xpack.esql.index.IndexResolution;
1213
import org.elasticsearch.xpack.esql.inference.InferenceResolution;
1314
import org.elasticsearch.xpack.esql.session.Configuration;
15+
import org.elasticsearch.xpack.esql.session.EsqlSession;
1416

1517
import java.util.Map;
1618

@@ -20,17 +22,41 @@ public record AnalyzerContext(
2022
IndexResolution indexResolution,
2123
Map<String, IndexResolution> lookupResolution,
2224
EnrichResolution enrichResolution,
23-
InferenceResolution inferenceResolution
25+
InferenceResolution inferenceResolution,
26+
TransportVersion minimumVersion
2427
) {
25-
// Currently for tests only, since most do not test lookups
26-
// TODO: make this even simpler, remove the enrichResolution for tests that do not require it (most tests)
28+
2729
public AnalyzerContext(
2830
Configuration configuration,
2931
EsqlFunctionRegistry functionRegistry,
3032
IndexResolution indexResolution,
33+
Map<String, IndexResolution> lookupResolution,
3134
EnrichResolution enrichResolution,
32-
InferenceResolution inferenceResolution
35+
InferenceResolution inferenceResolution,
36+
TransportVersion minimumVersion
3337
) {
34-
this(configuration, functionRegistry, indexResolution, Map.of(), enrichResolution, inferenceResolution);
38+
this.configuration = configuration;
39+
this.functionRegistry = functionRegistry;
40+
this.indexResolution = indexResolution;
41+
this.lookupResolution = lookupResolution;
42+
this.enrichResolution = enrichResolution;
43+
this.inferenceResolution = inferenceResolution;
44+
this.minimumVersion = minimumVersion;
45+
46+
assert minimumVersion != null : "AnalyzerContext must have a minimum transport version";
47+
assert minimumVersion.onOrBefore(TransportVersion.current())
48+
: "AnalyzerContext [" + minimumVersion + "] is not on or before current transport version [" + TransportVersion.current() + "]";
49+
}
50+
51+
public AnalyzerContext(Configuration configuration, EsqlFunctionRegistry functionRegistry, EsqlSession.PreAnalysisResult result) {
52+
this(
53+
configuration,
54+
functionRegistry,
55+
result.indices(),
56+
result.lookupIndices(),
57+
result.enrichResolution(),
58+
result.inferenceResolution(),
59+
result.minimumTransportVersion()
60+
);
3561
}
3662
}

0 commit comments

Comments
 (0)