Skip to content

Commit 9352610

Browse files
craigtavernercraigtaverner
committed
Multi-relation support, a Pre-requisite for views and sub-queries
This is extracted from the views prototype, which also includes necessary support for sub-queries.
1 parent ef5ab4e commit 9352610

35 files changed

+849
-459
lines changed

‎benchmarks/src/main/java/org/elasticsearch/benchmark/_nightly/esql/QueryPlanningBenchmark.java‎

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -14,8 +14,13 @@
1414
import org.elasticsearch.common.settings.Settings;
1515
import org.elasticsearch.index.IndexMode;
1616
import org.elasticsearch.license.XPackLicenseState;
17-
import org.elasticsearch.xpack.esql.analysis.*;
17+
import org.elasticsearch.xpack.esql.analysis.Analyzer;
18+
import org.elasticsearch.xpack.esql.analysis.AnalyzerContext;
19+
import org.elasticsearch.xpack.esql.analysis.AnalyzerSettings;
20+
import org.elasticsearch.xpack.esql.analysis.EnrichResolution;
21+
import org.elasticsearch.xpack.esql.analysis.Verifier;
1822
import org.elasticsearch.xpack.esql.core.expression.FoldContext;
23+
import org.elasticsearch.xpack.esql.core.tree.Source;
1924
import org.elasticsearch.xpack.esql.core.type.EsField;
2025
import org.elasticsearch.xpack.esql.core.util.DateUtils;
2126
import org.elasticsearch.xpack.esql.expression.function.EsqlFunctionRegistry;
@@ -26,6 +31,7 @@
2631
import org.elasticsearch.xpack.esql.optimizer.LogicalPlanOptimizer;
2732
import org.elasticsearch.xpack.esql.parser.EsqlParser;
2833
import org.elasticsearch.xpack.esql.parser.QueryParams;
34+
import org.elasticsearch.xpack.esql.plan.IndexPattern;
2935
import org.elasticsearch.xpack.esql.plan.logical.LogicalPlan;
3036
import org.elasticsearch.xpack.esql.plugin.QueryPragmas;
3137
import org.elasticsearch.xpack.esql.session.Configuration;
@@ -108,7 +114,7 @@ public void setup() {
108114
new AnalyzerContext(
109115
config,
110116
functionRegistry,
111-
IndexResolution.valid(esIndex),
117+
Map.of(new IndexPattern(Source.EMPTY, esIndex.name()), IndexResolution.valid(esIndex)),
112118
Map.of(),
113119
new EnrichResolution(),
114120
InferenceResolution.EMPTY,

‎x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/CsvTestsDataLoader.java‎

Lines changed: 40 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -78,8 +78,9 @@ public class CsvTestsDataLoader {
7878
private static final TestDataset LANGUAGES = new TestDataset("languages");
7979
private static final TestDataset LANGUAGES_LOOKUP = LANGUAGES.withIndex("languages_lookup").withSetting("lookup-settings.json");
8080
private static final TestDataset LANGUAGES_NON_UNIQUE_KEY = new TestDataset("languages_non_unique_key");
81-
private static final TestDataset LANGUAGES_LOOKUP_NON_UNIQUE_KEY = LANGUAGES_NON_UNIQUE_KEY.withIndex("languages_lookup_non_unique_key")
82-
.withSetting("lookup-settings.json");
81+
private static final TestDataset LANGUAGES_LOOKUP_NON_UNIQUE_KEY = LANGUAGES_LOOKUP.withIndex("languages_lookup_non_unique_key")
82+
.withData("languages_non_unique_key.csv")
83+
.withDynamicTypeMapping(Map.of("country", "text"));
8384
private static final TestDataset LANGUAGES_NESTED_FIELDS = new TestDataset(
8485
"languages_nested_fields",
8586
"mapping-languages_nested_fields.json",
@@ -423,11 +424,13 @@ private static void loadDataSetIntoEs(
423424
Logger logger = LogManager.getLogger(CsvTestsDataLoader.class);
424425

425426
Set<String> loadedDatasets = new HashSet<>();
427+
logger.info("Loading test datasets");
426428
for (var dataset : availableDatasetsForEs(supportsIndexModeLookup, supportsSourceFieldMapping, inferenceEnabled, timeSeriesOnly)) {
427429
load(client, dataset, logger, indexCreator);
428430
loadedDatasets.add(dataset.indexName);
429431
}
430432
forceMerge(client, loadedDatasets, logger);
433+
logger.info("Loading enrich policies");
431434
for (var policy : ENRICH_POLICIES) {
432435
loadEnrichPolicy(client, policy.policyName, policy.policyFileName, logger);
433436
}
@@ -569,6 +572,7 @@ private static void deleteInferenceEndpoint(RestClient client, String inferenceI
569572
}
570573

571574
private static void loadEnrichPolicy(RestClient client, String policyName, String policyFileName, Logger logger) throws IOException {
575+
logger.info("Loading enrich policy [{}] from file [{}]", policyName, policyFileName);
572576
URL policyMapping = getResource("/" + policyFileName);
573577
String entity = readTextFile(policyMapping);
574578
Request request = new Request("PUT", "/_enrich/policy/" + policyName);
@@ -588,6 +592,7 @@ private static URL getResource(String name) {
588592
}
589593

590594
private static void load(RestClient client, TestDataset dataset, Logger logger, IndexCreator indexCreator) throws IOException {
595+
logger.info("Loading dataset [{}] into ES index [{}]", dataset.dataFileName, dataset.indexName);
591596
URL mapping = getResource("/" + dataset.mappingFileName);
592597
Settings indexSettings = dataset.readSettingsFile();
593598
indexCreator.createIndex(client, dataset.indexName, readMappingFile(mapping, dataset.typeMapping), indexSettings);
@@ -854,15 +859,16 @@ public record TestDataset(
854859
String dataFileName,
855860
String settingFileName,
856861
boolean allowSubFields,
857-
@Nullable Map<String, String> typeMapping,
862+
@Nullable Map<String, String> typeMapping, // Override mappings read from mappings file
863+
@Nullable Map<String, String> dynamicTypeMapping, // Define mappings not in the mapping files, but available from field-caps
858864
boolean requiresInferenceEndpoint
859865
) {
860866
public TestDataset(String indexName, String mappingFileName, String dataFileName) {
861-
this(indexName, mappingFileName, dataFileName, null, true, null, false);
867+
this(indexName, mappingFileName, dataFileName, null, true, null, null, false);
862868
}
863869

864870
public TestDataset(String indexName) {
865-
this(indexName, "mapping-" + indexName + ".json", indexName + ".csv", null, true, null, false);
871+
this(indexName, "mapping-" + indexName + ".json", indexName + ".csv", null, true, null, null, false);
866872
}
867873

868874
public TestDataset withIndex(String indexName) {
@@ -873,6 +879,7 @@ public TestDataset withIndex(String indexName) {
873879
settingFileName,
874880
allowSubFields,
875881
typeMapping,
882+
dynamicTypeMapping,
876883
requiresInferenceEndpoint
877884
);
878885
}
@@ -885,6 +892,7 @@ public TestDataset withData(String dataFileName) {
885892
settingFileName,
886893
allowSubFields,
887894
typeMapping,
895+
dynamicTypeMapping,
888896
requiresInferenceEndpoint
889897
);
890898
}
@@ -897,6 +905,7 @@ public TestDataset noData() {
897905
settingFileName,
898906
allowSubFields,
899907
typeMapping,
908+
dynamicTypeMapping,
900909
requiresInferenceEndpoint
901910
);
902911
}
@@ -909,6 +918,7 @@ public TestDataset withSetting(String settingFileName) {
909918
settingFileName,
910919
allowSubFields,
911920
typeMapping,
921+
dynamicTypeMapping,
912922
requiresInferenceEndpoint
913923
);
914924
}
@@ -921,6 +931,7 @@ public TestDataset noSubfields() {
921931
settingFileName,
922932
false,
923933
typeMapping,
934+
dynamicTypeMapping,
924935
requiresInferenceEndpoint
925936
);
926937
}
@@ -933,12 +944,35 @@ public TestDataset withTypeMapping(Map<String, String> typeMapping) {
933944
settingFileName,
934945
allowSubFields,
935946
typeMapping,
947+
dynamicTypeMapping,
948+
requiresInferenceEndpoint
949+
);
950+
}
951+
952+
public TestDataset withDynamicTypeMapping(Map<String, String> dynamicTypeMapping) {
953+
return new TestDataset(
954+
indexName,
955+
mappingFileName,
956+
dataFileName,
957+
settingFileName,
958+
allowSubFields,
959+
typeMapping,
960+
dynamicTypeMapping,
936961
requiresInferenceEndpoint
937962
);
938963
}
939964

940965
public TestDataset withInferenceEndpoint(boolean needsInference) {
941-
return new TestDataset(indexName, mappingFileName, dataFileName, settingFileName, allowSubFields, typeMapping, needsInference);
966+
return new TestDataset(
967+
indexName,
968+
mappingFileName,
969+
dataFileName,
970+
settingFileName,
971+
allowSubFields,
972+
typeMapping,
973+
dynamicTypeMapping,
974+
needsInference
975+
);
942976
}
943977

944978
private Settings readSettingsFile() throws IOException {

‎x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/EsqlTestUtils.java‎

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -99,6 +99,7 @@
9999
import org.elasticsearch.xpack.esql.inference.InferenceService;
100100
import org.elasticsearch.xpack.esql.optimizer.LogicalOptimizerContext;
101101
import org.elasticsearch.xpack.esql.parser.QueryParam;
102+
import org.elasticsearch.xpack.esql.plan.IndexPattern;
102103
import org.elasticsearch.xpack.esql.plan.logical.Enrich;
103104
import org.elasticsearch.xpack.esql.plan.logical.EsRelation;
104105
import org.elasticsearch.xpack.esql.plan.logical.Limit;
@@ -441,11 +442,11 @@ public static TransportVersion randomMinimumVersion() {
441442
public static AnalyzerContext testAnalyzerContext(
442443
Configuration configuration,
443444
EsqlFunctionRegistry functionRegistry,
444-
IndexResolution indexResolution,
445+
Map<IndexPattern, IndexResolution> indexResolutions,
445446
EnrichResolution enrichResolution,
446447
InferenceResolution inferenceResolution
447448
) {
448-
return testAnalyzerContext(configuration, functionRegistry, indexResolution, Map.of(), enrichResolution, inferenceResolution);
449+
return testAnalyzerContext(configuration, functionRegistry, indexResolutions, Map.of(), enrichResolution, inferenceResolution);
449450
}
450451

451452
/**
@@ -454,15 +455,15 @@ public static AnalyzerContext testAnalyzerContext(
454455
public static AnalyzerContext testAnalyzerContext(
455456
Configuration configuration,
456457
EsqlFunctionRegistry functionRegistry,
457-
IndexResolution indexResolution,
458+
Map<IndexPattern, IndexResolution> indexResolutions,
458459
Map<String, IndexResolution> lookupResolution,
459460
EnrichResolution enrichResolution,
460461
InferenceResolution inferenceResolution
461462
) {
462463
return new AnalyzerContext(
463464
configuration,
464465
functionRegistry,
465-
indexResolution,
466+
indexResolutions,
466467
lookupResolution,
467468
enrichResolution,
468469
inferenceResolution,

‎x-pack/plugin/esql/qa/testFixtures/src/main/resources/inlinestats.csv-spec‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3976,14 +3976,15 @@ required_capability: inline_stats
39763976
required_capability: fix_join_output_merging
39773977

39783978
FROM languages_lookup_non_unique_key
3979+
| EVAL country = MV_SORT(country)
39793980
| KEEP country, language_name
39803981
| EVAL language_code = null::integer
39813982
| INLINE STATS MAX(language_code) BY language_code
39823983
| SORT country
39833984
| LIMIT 5
39843985
;
39853986

3986-
country:text |language_name:keyword |MAX(language_code):integer |language_code:integer
3987+
country:keyword |language_name:keyword |MAX(language_code):integer |language_code:integer
39873988
Atlantis |null |null |null
39883989
[Austria, Germany]|German |null |null
39893990
Canada |English |null |null

‎x-pack/plugin/esql/qa/testFixtures/src/main/resources/lookup-join.csv-spec‎

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -247,11 +247,12 @@ FROM employees
247247
| EVAL language_code = emp_no % 10
248248
| LOOKUP JOIN languages_lookup_non_unique_key ON language_code
249249
| WHERE emp_no > 10090 AND emp_no < 10096
250+
| EVAL country = MV_SORT(country)
250251
| SORT emp_no, country
251252
| KEEP emp_no, language_code, language_name, country
252253
;
253254

254-
emp_no:integer | language_code:integer | language_name:keyword | country:text
255+
emp_no:integer | language_code:integer | language_name:keyword | country:keyword
255256
10091 | 1 | English | Canada
256257
10091 | 1 | null | United Kingdom
257258
10091 | 1 | English | United States of America
@@ -272,11 +273,12 @@ FROM employees
272273
| LIMIT 5
273274
| EVAL language_code = emp_no % 10
274275
| LOOKUP JOIN languages_lookup_non_unique_key ON language_code
276+
| EVAL country = MV_SORT(country)
275277
| KEEP emp_no, language_code, language_name, country
276278
;
277279

278280
ignoreOrder:true
279-
emp_no:integer | language_code:integer | language_name:keyword | country:text
281+
emp_no:integer | language_code:integer | language_name:keyword | country:keyword
280282
10001 | 1 | English | Canada
281283
10001 | 1 | English | null
282284
10001 | 1 | null | United Kingdom
@@ -324,7 +326,7 @@ ROW language_code = 2
324326

325327
ignoreOrder:true
326328
language_code:integer | country:text | language_name:keyword
327-
2 | [Austria, Germany] | German
329+
2 | [Germany, Austria] | German
328330
2 | Switzerland | German
329331
2 | null | German
330332
;

‎x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java‎

Lines changed: 15 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -247,12 +247,10 @@ private static class ResolveTable extends ParameterizedAnalyzerRule<UnresolvedRe
247247

248248
@Override
249249
protected LogicalPlan rule(UnresolvedRelation plan, AnalyzerContext context) {
250-
return resolveIndex(
251-
plan,
252-
plan.indexMode().equals(IndexMode.LOOKUP)
253-
? context.lookupResolution().get(plan.indexPattern().indexPattern())
254-
: context.indexResolution()
255-
);
250+
IndexResolution indexResolution = plan.indexMode().equals(IndexMode.LOOKUP)
251+
? context.lookupResolution().get(plan.indexPattern().indexPattern())
252+
: context.indexResolution().get(plan.indexPattern());
253+
return resolveIndex(plan, indexResolution);
256254
}
257255

258256
private LogicalPlan resolveIndex(UnresolvedRelation plan, IndexResolution indexResolution) {
@@ -270,6 +268,7 @@ private LogicalPlan resolveIndex(UnresolvedRelation plan, IndexResolution indexR
270268
plan.telemetryLabel()
271269
);
272270
}
271+
// assert indexResolution.matches(plan.indexPattern().indexPattern()) : "Expected index resolution to match the index pattern";
273272
IndexPattern table = plan.indexPattern();
274273
if (indexResolution.matches(table.indexPattern()) == false) {
275274
// TODO: fix this (and tests), or drop check (seems SQL-inherited, where's also defective)
@@ -537,7 +536,7 @@ protected LogicalPlan rule(LogicalPlan plan, AnalyzerContext context) {
537536
}
538537

539538
if (plan instanceof Insist i) {
540-
return resolveInsist(i, childrenOutput, context.indexResolution());
539+
return resolveInsist(i, childrenOutput, context.indexResolution().values());
541540
}
542541

543542
if (plan instanceof Fuse fuse) {
@@ -958,23 +957,29 @@ private List<Attribute> resolveUsingColumns(List<Attribute> cols, List<Attribute
958957
return resolved;
959958
}
960959

961-
private LogicalPlan resolveInsist(Insist insist, List<Attribute> childrenOutput, IndexResolution indexResolution) {
960+
private LogicalPlan resolveInsist(Insist insist, List<Attribute> childrenOutput, Collection<IndexResolution> indexResolution) {
962961
List<Attribute> list = new ArrayList<>();
963962
for (Attribute a : insist.insistedAttributes()) {
964963
list.add(resolveInsistAttribute(a, childrenOutput, indexResolution));
965964
}
966965
return insist.withAttributes(list);
967966
}
968967

969-
private Attribute resolveInsistAttribute(Attribute attribute, List<Attribute> childrenOutput, IndexResolution indexResolution) {
968+
private Attribute resolveInsistAttribute(
969+
Attribute attribute,
970+
List<Attribute> childrenOutput,
971+
Collection<IndexResolution> indexResolution
972+
) {
970973
Attribute resolvedCol = maybeResolveAttribute((UnresolvedAttribute) attribute, childrenOutput);
971974
// Field isn't mapped anywhere.
972975
if (resolvedCol instanceof UnresolvedAttribute) {
973976
return insistKeyword(attribute);
974977
}
975978

976979
// Field is partially unmapped.
977-
if (resolvedCol instanceof FieldAttribute fa && indexResolution.get().isPartiallyUnmappedField(fa.name())) {
980+
// TODO: Should the check for partially unmapped fields be done specific to each sub-query in a fork?
981+
if (resolvedCol instanceof FieldAttribute fa
982+
&& indexResolution.stream().anyMatch(r -> r.get().isPartiallyUnmappedField(fa.name()))) {
978983
return fa.dataType() == KEYWORD ? insistKeyword(fa) : invalidInsistAttribute(fa);
979984
}
980985

‎x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/AnalyzerContext.java‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@
1111
import org.elasticsearch.xpack.esql.expression.function.EsqlFunctionRegistry;
1212
import org.elasticsearch.xpack.esql.index.IndexResolution;
1313
import org.elasticsearch.xpack.esql.inference.InferenceResolution;
14+
import org.elasticsearch.xpack.esql.plan.IndexPattern;
1415
import org.elasticsearch.xpack.esql.session.Configuration;
1516
import org.elasticsearch.xpack.esql.session.EsqlSession;
1617

@@ -19,7 +20,7 @@
1920
public record AnalyzerContext(
2021
Configuration configuration,
2122
EsqlFunctionRegistry functionRegistry,
22-
IndexResolution indexResolution,
23+
Map<IndexPattern, IndexResolution> indexResolution,
2324
Map<String, IndexResolution> lookupResolution,
2425
EnrichResolution enrichResolution,
2526
InferenceResolution inferenceResolution,
@@ -29,7 +30,7 @@ public record AnalyzerContext(
2930
public AnalyzerContext(
3031
Configuration configuration,
3132
EsqlFunctionRegistry functionRegistry,
32-
IndexResolution indexResolution,
33+
Map<IndexPattern, IndexResolution> indexResolution,
3334
Map<String, IndexResolution> lookupResolution,
3435
EnrichResolution enrichResolution,
3536
InferenceResolution inferenceResolution,
@@ -52,7 +53,7 @@ public AnalyzerContext(Configuration configuration, EsqlFunctionRegistry functio
5253
this(
5354
configuration,
5455
functionRegistry,
55-
result.indices(),
56+
result.indexResolutions(),
5657
result.lookupIndices(),
5758
result.enrichResolution(),
5859
result.inferenceResolution(),

0 commit comments

Comments
 (0)