Skip to content

Commit 22569b8

Browse files
GalLaloucheGalLalouche
committed
Things pass! Excitement!
1 parent 1204791 commit 22569b8

File tree

20 files changed

+380
-169
lines changed

20 files changed

+380
-169
lines changed

‎x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/util/CollectionUtils.java‎

Lines changed: 0 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -10,9 +10,7 @@
1010
import java.util.Collection;
1111
import java.util.Collections;
1212
import java.util.List;
13-
import java.util.OptionalInt;
1413
import java.util.Set;
15-
import java.util.function.Predicate;
1614

1715
import static java.util.Collections.emptyList;
1816

@@ -85,13 +83,4 @@ public static <T> List<T> nullSafeList(T... entries) {
8583
}
8684
return list;
8785
}
88-
89-
public static <T> OptionalInt findFirstIndex(List<T> list, Predicate<T> predicate) {
90-
for (int i = 0; i < list.size(); i++) {
91-
if (predicate.test(list.get(i))) {
92-
return OptionalInt.of(i);
93-
}
94-
}
95-
return OptionalInt.empty();
96-
}
9786
}

‎x-pack/plugin/esql/qa/testFixtures/src/main/resources/data/partial_mapping_sample_data.csv‎

Lines changed: 8 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,8 @@
1-
@timestamp:date,client_ip:ip,event_duration:long,message:keyword,unmapped_message:keyword,unmapped_event_duration:keyword
2-
2024-10-23T13:55:01.543Z,173.21.3.15,1756466,Connected to 10.1.0.1!,Disconnected from 10.1.0.1,1756468
3-
2024-10-23T13:53:55.832Z,173.21.3.15,5033754,Connection error?,Disconnection error,5033756
4-
2024-10-23T13:52:55.015Z,173.21.3.15,8268152,Connection error?,Disconnection error,8268154
5-
2024-10-23T13:51:54.732Z,173.21.3.15,725447,Connection error?,Disconnection error,725449
6-
2024-10-23T13:33:34.937Z,173.21.0.5,1232381,42,43,1232383
7-
2024-10-23T12:27:28.948Z,173.21.2.113,2764888,Connected to 10.1.0.2!,Disconnected from 10.1.0.2,2764890
8-
2024-10-23T12:15:03.360Z,173.21.2.162,3450232,Connected to 10.1.0.3!,Disconnected from 10.1.0.3,3450234
1+
@timestamp:date,client_ip:ip,event_duration:long,message:keyword,unmapped_message:keyword,unmapped_event_duration:keyword,unmapped.nested:keyword
2+
2024-10-23T13:55:01.543Z,173.21.3.15,1756466,Connected to 10.1.0.1!,Disconnected from 10.1.0.1,1756468,a
3+
2024-10-23T13:53:55.832Z,173.21.3.15,5033754,Connection error?,Disconnection error,5033756,b
4+
2024-10-23T13:52:55.015Z,173.21.3.15,8268152,Connection error?,Disconnection error,8268154,c
5+
2024-10-23T13:51:54.732Z,173.21.3.15,725447,Connection error?,Disconnection error,725449,d
6+
2024-10-23T13:33:34.937Z,173.21.0.5,1232381,42,43,1232383,e
7+
2024-10-23T12:27:28.948Z,173.21.2.113,2764888,Connected to 10.1.0.2!,Disconnected from 10.1.0.2,2764890,f
8+
2024-10-23T12:15:03.360Z,173.21.2.162,3450232,Connected to 10.1.0.3!,Disconnected from 10.1.0.3,3450234,g

‎x-pack/plugin/esql/qa/testFixtures/src/main/resources/unmapped_fields.csv-spec‎

Lines changed: 185 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -138,6 +138,167 @@ FROM partial_mapping_excluded_source_sample_data
138138
2024-10-23T12:15:03.360Z | null
139139
;
140140

141+
fieldIsNestedAndMapped
142+
required_capability: unmapped_fields
143+
FROM addresses
144+
| INSIST_🐔 city.name
145+
| KEEP city.name
146+
| SORT city.name DESC
147+
;
148+
149+
city.name:keyword
150+
Tokyo
151+
San Francisco
152+
Amsterdam
153+
;
154+
155+
fieldIsNestedAndUnmapped
156+
required_capability: unmapped_fields
157+
FROM partial_mapping_sample_data
158+
| INSIST_🐔 unmapped.nested
159+
| KEEP @timestamp, unmapped.nested
160+
| SORT @timestamp
161+
;
162+
163+
@timestamp:date | unmapped.nested:keyword
164+
2024-10-23T12:15:03.360Z | g
165+
2024-10-23T12:27:28.948Z | f
166+
2024-10-23T13:33:34.937Z | e
167+
2024-10-23T13:51:54.732Z | d
168+
2024-10-23T13:52:55.015Z | c
169+
2024-10-23T13:53:55.832Z | b
170+
2024-10-23T13:55:01.543Z | a
171+
;
172+
173+
fieldIsNestedAndNonExistent
174+
required_capability: unmapped_fields
175+
FROM partial_mapping_sample_data
176+
| INSIST_🐔 unmapped.nested.nonexistent
177+
| KEEP @timestamp, unmapped.nested.nonexistent
178+
| SORT @timestamp
179+
;
180+
181+
@timestamp:date | unmapped.nested.nonexistent:keyword
182+
2024-10-23T12:15:03.360Z | null
183+
2024-10-23T12:27:28.948Z | null
184+
2024-10-23T13:33:34.937Z | null
185+
2024-10-23T13:51:54.732Z | null
186+
2024-10-23T13:52:55.015Z | null
187+
2024-10-23T13:53:55.832Z | null
188+
2024-10-23T13:55:01.543Z | null
189+
;
190+
191+
#########################
192+
# Multi-parameter tests #
193+
#########################
194+
195+
noFieldExistsMultiParametersSingleIndex
196+
required_capability: unmapped_fields
197+
FROM partial_mapping_sample_data
198+
| INSIST_🐔 foo, bar, bazz
199+
| KEEP @timestamp, foo, bar, bazz
200+
| SORT @timestamp DESC
201+
;
202+
203+
@timestamp:date | foo:keyword | bar:keyword | bazz:keyword
204+
2024-10-23T13:55:01.543Z | null | null | null
205+
2024-10-23T13:53:55.832Z | null | null | null
206+
2024-10-23T13:52:55.015Z | null | null | null
207+
2024-10-23T13:51:54.732Z | null | null | null
208+
2024-10-23T13:33:34.937Z | null | null | null
209+
2024-10-23T12:27:28.948Z | null | null | null
210+
2024-10-23T12:15:03.360Z | null | null | null
211+
;
212+
213+
mixedFieldsMultiParametersSingleIndex
214+
required_capability: unmapped_fields
215+
FROM partial_mapping_sample_data
216+
| INSIST_🐔 foo, message, unmapped_message
217+
| KEEP @timestamp, foo, message, unmapped_message
218+
| SORT @timestamp DESC
219+
;
220+
221+
@timestamp:date | foo:keyword | message:keyword | unmapped_message:keyword
222+
2024-10-23T13:55:01.543Z | null | Connected to 10.1.0.1! | Disconnected from 10.1.0.1
223+
2024-10-23T13:53:55.832Z | null | Connection error? | Disconnection error
224+
2024-10-23T13:52:55.015Z | null | Connection error? | Disconnection error
225+
2024-10-23T13:51:54.732Z | null | Connection error? | Disconnection error
226+
2024-10-23T13:33:34.937Z | null | 42 | 43
227+
2024-10-23T12:27:28.948Z | null | Connected to 10.1.0.2! | Disconnected from 10.1.0.2
228+
2024-10-23T12:15:03.360Z | null | Connected to 10.1.0.3! | Disconnected from 10.1.0.3
229+
;
230+
231+
repeatedInsistFieldsAreSilentlyIgnored
232+
required_capability: unmapped_fields
233+
FROM partial_mapping_sample_data
234+
| INSIST_🐔 unmapped_message, foo, message, foo, message, unmapped_message
235+
| KEEP @timestamp, foo, message, unmapped_message
236+
| SORT @timestamp DESC
237+
;
238+
239+
@timestamp:date | foo:keyword | message:keyword | unmapped_message:keyword
240+
2024-10-23T13:55:01.543Z | null | Connected to 10.1.0.1! | Disconnected from 10.1.0.1
241+
2024-10-23T13:53:55.832Z | null | Connection error? | Disconnection error
242+
2024-10-23T13:52:55.015Z | null | Connection error? | Disconnection error
243+
2024-10-23T13:51:54.732Z | null | Connection error? | Disconnection error
244+
2024-10-23T13:33:34.937Z | null | 42 | 43
245+
2024-10-23T12:27:28.948Z | null | Connected to 10.1.0.2! | Disconnected from 10.1.0.2
246+
2024-10-23T12:15:03.360Z | null | Connected to 10.1.0.3! | Disconnected from 10.1.0.3
247+
;
248+
249+
mixedFieldsMultiParametersMultiIndex
250+
required_capability: unmapped_fields
251+
required_capability: index_metadata_field
252+
FROM partial_mapping_sample_data, sample_data METADATA _index
253+
| INSIST_🐔 foo, message, unmapped_message
254+
| KEEP _index, @timestamp, foo, message, unmapped_message
255+
| SORT @timestamp DESC
256+
;
257+
258+
_index:keyword | @timestamp:datetime | foo:keyword | message:keyword | unmapped_message:keyword
259+
partial_mapping_sample_data | 2024-10-23T13:55:01.543Z | null | Connected to 10.1.0.1! | Disconnected from 10.1.0.1
260+
partial_mapping_sample_data | 2024-10-23T13:53:55.832Z | null | Connection error? | Disconnection error
261+
partial_mapping_sample_data | 2024-10-23T13:52:55.015Z | null | Connection error? | Disconnection error
262+
partial_mapping_sample_data | 2024-10-23T13:51:54.732Z | null | Connection error? | Disconnection error
263+
partial_mapping_sample_data | 2024-10-23T13:33:34.937Z | null | 42 | 43
264+
partial_mapping_sample_data | 2024-10-23T12:27:28.948Z | null | Connected to 10.1.0.2! | Disconnected from 10.1.0.2
265+
partial_mapping_sample_data | 2024-10-23T12:15:03.360Z | null | Connected to 10.1.0.3! | Disconnected from 10.1.0.3
266+
sample_data | 2023-10-23T13:55:01.543Z | null | Connected to 10.1.0.1 | null
267+
sample_data | 2023-10-23T13:53:55.832Z | null | Connection error | null
268+
sample_data | 2023-10-23T13:52:55.015Z | null | Connection error | null
269+
sample_data | 2023-10-23T13:51:54.732Z | null | Connection error | null
270+
sample_data | 2023-10-23T13:33:34.937Z | null | Disconnected | null
271+
sample_data | 2023-10-23T12:27:28.948Z | null | Connected to 10.1.0.2 | null
272+
sample_data | 2023-10-23T12:15:03.360Z | null | Connected to 10.1.0.3 | null
273+
;
274+
275+
insistOnTopOfInsistMultiIndex
276+
required_capability: unmapped_fields
277+
required_capability: index_metadata_field
278+
FROM partial_mapping_sample_data, sample_data METADATA _index
279+
| INSIST_🐔 foo, message
280+
| INSIST_🐔 unmapped_message
281+
| KEEP _index, @timestamp, foo, message, unmapped_message
282+
| SORT @timestamp DESC
283+
;
284+
285+
_index:keyword | @timestamp:datetime | foo:keyword | message:keyword | unmapped_message:keyword
286+
partial_mapping_sample_data | 2024-10-23T13:55:01.543Z | null | Connected to 10.1.0.1! | Disconnected from 10.1.0.1
287+
partial_mapping_sample_data | 2024-10-23T13:53:55.832Z | null | Connection error? | Disconnection error
288+
partial_mapping_sample_data | 2024-10-23T13:52:55.015Z | null | Connection error? | Disconnection error
289+
partial_mapping_sample_data | 2024-10-23T13:51:54.732Z | null | Connection error? | Disconnection error
290+
partial_mapping_sample_data | 2024-10-23T13:33:34.937Z | null | 42 | 43
291+
partial_mapping_sample_data | 2024-10-23T12:27:28.948Z | null | Connected to 10.1.0.2! | Disconnected from 10.1.0.2
292+
partial_mapping_sample_data | 2024-10-23T12:15:03.360Z | null | Connected to 10.1.0.3! | Disconnected from 10.1.0.3
293+
sample_data | 2023-10-23T13:55:01.543Z | null | Connected to 10.1.0.1 | null
294+
sample_data | 2023-10-23T13:53:55.832Z | null | Connection error | null
295+
sample_data | 2023-10-23T13:52:55.015Z | null | Connection error | null
296+
sample_data | 2023-10-23T13:51:54.732Z | null | Connection error | null
297+
sample_data | 2023-10-23T13:33:34.937Z | null | Disconnected | null
298+
sample_data | 2023-10-23T12:27:28.948Z | null | Connected to 10.1.0.2 | null
299+
sample_data | 2023-10-23T12:15:03.360Z | null | Connected to 10.1.0.3 | null
300+
;
301+
141302
#####################
142303
# Multi index tests #
143304
#####################
@@ -152,20 +313,20 @@ FROM partial_mapping_sample_data, sample_data METADATA _index
152313
;
153314

154315
_index:keyword | @timestamp:date | foo:keyword
155-
partial_mapping_sample_data | 2024-10-23T13:55:01.543Z | null
156-
partial_mapping_sample_data | 2024-10-23T13:53:55.832Z | null
157-
partial_mapping_sample_data | 2024-10-23T13:52:55.015Z | null
158-
partial_mapping_sample_data | 2024-10-23T13:51:54.732Z | null
159-
partial_mapping_sample_data | 2024-10-23T13:33:34.937Z | null
160-
partial_mapping_sample_data | 2024-10-23T12:27:28.948Z | null
161-
partial_mapping_sample_data | 2024-10-23T12:15:03.360Z | null
162-
sample_data | 2023-10-23T13:55:01.543Z | null
163-
sample_data | 2023-10-23T13:53:55.832Z | null
164-
sample_data | 2023-10-23T13:52:55.015Z | null
165-
sample_data | 2023-10-23T13:51:54.732Z | null
166-
sample_data | 2023-10-23T13:33:34.937Z | null
167-
sample_data | 2023-10-23T12:27:28.948Z | null
168-
sample_data | 2023-10-23T12:15:03.360Z | null
316+
partial_mapping_sample_data | 2024-10-23T13:55:01.543Z | null
317+
partial_mapping_sample_data | 2024-10-23T13:53:55.832Z | null
318+
partial_mapping_sample_data | 2024-10-23T13:52:55.015Z | null
319+
partial_mapping_sample_data | 2024-10-23T13:51:54.732Z | null
320+
partial_mapping_sample_data | 2024-10-23T13:33:34.937Z | null
321+
partial_mapping_sample_data | 2024-10-23T12:27:28.948Z | null
322+
partial_mapping_sample_data | 2024-10-23T12:15:03.360Z | null
323+
sample_data | 2023-10-23T13:55:01.543Z | null
324+
sample_data | 2023-10-23T13:53:55.832Z | null
325+
sample_data | 2023-10-23T13:52:55.015Z | null
326+
sample_data | 2023-10-23T13:51:54.732Z | null
327+
sample_data | 2023-10-23T13:33:34.937Z | null
328+
sample_data | 2023-10-23T12:27:28.948Z | null
329+
sample_data | 2023-10-23T12:15:03.360Z | null
169330
;
170331

171332
fieldIsUnmappedMultiIndex
@@ -312,19 +473,19 @@ FROM partial_mapping_sample_data,partial_mapping_no_source_sample_data METADATA
312473
;
313474

314475
_index:keyword | @timestamp:date | message:keyword
315-
partial_mapping_no_source_sample_data | 2024-10-23T12:15:03.360Z | null
316-
partial_mapping_no_source_sample_data | 2024-10-23T12:27:28.948Z | null
317-
partial_mapping_no_source_sample_data | 2024-10-23T13:33:34.937Z | null
318-
partial_mapping_no_source_sample_data | 2024-10-23T13:51:54.732Z | null
319-
partial_mapping_no_source_sample_data | 2024-10-23T13:52:55.015Z | null
320-
partial_mapping_no_source_sample_data | 2024-10-23T13:53:55.832Z | null
476+
partial_mapping_no_source_sample_data | 2024-10-23T12:15:03.360Z | null
477+
partial_mapping_no_source_sample_data | 2024-10-23T12:27:28.948Z | null
478+
partial_mapping_no_source_sample_data | 2024-10-23T13:33:34.937Z | null
479+
partial_mapping_no_source_sample_data | 2024-10-23T13:51:54.732Z | null
480+
partial_mapping_no_source_sample_data | 2024-10-23T13:52:55.015Z | null
481+
partial_mapping_no_source_sample_data | 2024-10-23T13:53:55.832Z | null
321482
partial_mapping_no_source_sample_data | 2024-10-23T13:55:01.543Z | null
322483
partial_mapping_sample_data | 2024-10-23T12:15:03.360Z | Connected to 10.1.0.3!
323484
partial_mapping_sample_data | 2024-10-23T12:27:28.948Z | Connected to 10.1.0.2!
324-
partial_mapping_sample_data | 2024-10-23T13:33:34.937Z | 42
325-
partial_mapping_sample_data | 2024-10-23T13:51:54.732Z | Connection error?
326-
partial_mapping_sample_data | 2024-10-23T13:52:55.015Z | Connection error?
327-
partial_mapping_sample_data | 2024-10-23T13:53:55.832Z | Connection error?
485+
partial_mapping_sample_data | 2024-10-23T13:33:34.937Z | 42
486+
partial_mapping_sample_data | 2024-10-23T13:51:54.732Z | Connection error?
487+
partial_mapping_sample_data | 2024-10-23T13:52:55.015Z | Connection error?
488+
partial_mapping_sample_data | 2024-10-23T13:53:55.832Z | Connection error?
328489
partial_mapping_sample_data | 2024-10-23T13:55:01.543Z | Connected to 10.1.0.1!
329490
;
330491

‎x-pack/plugin/esql/src/main/antlr/EsqlBaseLexer.g4‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -86,7 +86,7 @@ JOIN_LOOKUP : 'lookup' -> pushMode(JOIN_MODE);
8686
// main section while preserving alphabetical order:
8787
// MYCOMMAND : 'mycommand' -> ...
8888
DEV_INLINESTATS : {this.isDevVersion()}? 'inlinestats' -> pushMode(EXPRESSION_MODE);
89-
DEV_INSIST : {this.isDevVersion()}? 'insist_🐔' -> pushMode(EXPRESSION_MODE);
89+
DEV_INSIST : {this.isDevVersion()}? 'insist_🐔' -> pushMode(PROJECT_MODE);
9090
DEV_LOOKUP : {this.isDevVersion()}? 'lookup_🐔' -> pushMode(LOOKUP_MODE);
9191
DEV_METRICS : {this.isDevVersion()}? 'metrics' -> pushMode(METRICS_MODE);
9292
// list of all JOIN commands
@@ -310,7 +310,7 @@ FROM_WS
310310
;
311311
312312
//
313-
// DROP, KEEP
313+
// DROP, KEEP, INSIST
314314
//
315315
mode PROJECT_MODE;
316316
PROJECT_PIPE : PIPE -> type(PIPE), popMode;

‎x-pack/plugin/esql/src/main/antlr/EsqlBaseParser.g4‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -341,6 +341,6 @@ joinPredicate
341341
;
342342

343343
insistCommand
344-
: DEV_INSIST identifier
344+
: DEV_INSIST qualifiedNamePatterns
345345
;
346346

‎x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java‎

Lines changed: 13 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -685,15 +685,17 @@ private List<Attribute> resolveUsingColumns(List<Attribute> cols, List<Attribute
685685
}
686686

687687
private LogicalPlan resolveInsist(Insist insist, List<Attribute> childrenOutput, IndexResolution indexResolution) {
688-
return insist.withAttribute(resolveInsistAttribute(insist, childrenOutput, indexResolution));
688+
return insist.withAttributes(
689+
insist.insistedAttributes().stream().map(a -> resolveInsistAttribute(a, childrenOutput, indexResolution)).toList()
690+
);
689691
}
690692

691-
private Attribute resolveInsistAttribute(Insist insist, List<Attribute> childrenOutput, IndexResolution indexResolution) {
692-
assert insist.attribute() instanceof UnresolvedAttribute : "INSIST should be unresolved at this point";
693-
Attribute resolvedCol = maybeResolveAttribute((UnresolvedAttribute) insist.attribute(), childrenOutput);
693+
private Attribute resolveInsistAttribute(Attribute attribute, List<Attribute> childrenOutput, IndexResolution indexResolution) {
694+
assert attribute instanceof UnresolvedAttribute : "INSIST should be unresolved at this point";
695+
Attribute resolvedCol = maybeResolveAttribute((UnresolvedAttribute) attribute, childrenOutput);
694696
// Field isn't mapped anywhere.
695697
if (resolvedCol instanceof UnresolvedAttribute) {
696-
return insistKeyword(insist);
698+
return insistKeyword(attribute);
697699
}
698700

699701
assert resolvedCol instanceof FieldAttribute : "Resolved INSIST attribute should resolve to a field attribute";
@@ -702,31 +704,23 @@ private Attribute resolveInsistAttribute(Insist insist, List<Attribute> children
702704
// Field is partially unmapped.
703705
if (indexResolution.get().isPartiallyUnmappedField(name)) {
704706
return field.getDataType() == KEYWORD
705-
? insistKeyword(insist)
706-
: invalidInsistAttribute(insist, (FieldAttribute) resolvedCol);
707+
? insistKeyword(attribute)
708+
: invalidInsistAttribute(attribute, (FieldAttribute) resolvedCol);
707709
}
708710

709711
// Field is mapped everywhere; we can safely ignore the INSIST command.
710712
return resolvedCol;
711713
}
712714

713-
private static EsRelation mergeInsist(Insist insist, Function<List<Attribute>, List<Attribute>> updateAttributes) {
714-
assert insist.child() instanceof EsRelation : "INSIST should be on top of a relation (see LogicalPlanBuilder)";
715-
var relation = (EsRelation) insist.child();
716-
var newOutput = updateAttributes.apply(relation.output());
717-
return relation.withAttributes(newOutput);
718-
}
719-
720-
private static FieldAttribute invalidInsistAttribute(Insist insist, FieldAttribute fa) {
715+
private static FieldAttribute invalidInsistAttribute(Attribute attribute, FieldAttribute fa) {
721716
String name = fa.name();
722717
var messageFormat = "Cannot use field [%s] due to ambiguities caused by INSIST. "
723718
+ "Unmapped fields are treated as KEYWORD in unmapped indices, but field is mapped to another type";
724-
return new FieldAttribute(insist.source(), name, new InvalidMappedField(name, Strings.format(messageFormat, name)));
719+
return new FieldAttribute(attribute.source(), name, new InvalidMappedField(name, Strings.format(messageFormat, name)));
725720
}
726721

727-
private static FieldAttribute insistKeyword(Insist insist) {
728-
String name = insist.attribute().name();
729-
return new FieldAttribute(insist.source(), name, new PotentiallyUnmappedKeywordEsField(name));
722+
private static FieldAttribute insistKeyword(Attribute attribute) {
723+
return new FieldAttribute(attribute.source(), attribute.name(), new PotentiallyUnmappedKeywordEsField(attribute.name()));
730724
}
731725

732726
private Attribute maybeResolveAttribute(UnresolvedAttribute ua, List<Attribute> childrenOutput) {

0 commit comments

Comments
 (0)