Skip to content

Commit 5ac4d8c

Browse files
craigtavernercraigtaverner
authored
Fix union-types where one index is missing the field (#111932)
* Fix union-types where one index is missing the field When none of the indexes has the field, a validation error is correctly thrown, and when all indexes have the field, union-types works as normal. But when some indexes have the field and some do not, we were getting and internal error. We treat this case similarly to when some documents are missing the field, in which case `null` values are produced. So now a multi-index query where some indexes are missing the field will produce nulls for the documents coming from those indexes. * Update docs/changelog/111932.yaml * Added capability for this fix (missing-field)
1 parent 9387ce3 commit 5ac4d8c

File tree

7 files changed

+109
-1
lines changed

7 files changed

+109
-1
lines changed

‎docs/changelog/111932.yaml‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
pr: 111932
2+
summary: Fix union-types where one index is missing the field
3+
area: ES|QL
4+
type: bug
5+
issues:
6+
- 111912

‎x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/CsvTestsDataLoader.java‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -68,6 +68,11 @@ public class CsvTestsDataLoader {
6868
"mapping-sample_data_ts_long.json",
6969
"sample_data_ts_long.csv"
7070
);
71+
private static final TestsDataset MISSING_IP_SAMPLE_DATA = new TestsDataset(
72+
"missing_ip_sample_data",
73+
"mapping-missing_ip_sample_data.json",
74+
"missing_ip_sample_data.csv"
75+
);
7176
private static final TestsDataset CLIENT_IPS = new TestsDataset("clientips", "mapping-clientips.json", "clientips.csv");
7277
private static final TestsDataset CLIENT_CIDR = new TestsDataset("client_cidr", "mapping-client_cidr.json", "client_cidr.csv");
7378
private static final TestsDataset AGES = new TestsDataset("ages", "mapping-ages.json", "ages.csv");
@@ -112,6 +117,7 @@ public class CsvTestsDataLoader {
112117
Map.entry(ALERTS.indexName, ALERTS),
113118
Map.entry(SAMPLE_DATA_STR.indexName, SAMPLE_DATA_STR),
114119
Map.entry(SAMPLE_DATA_TS_LONG.indexName, SAMPLE_DATA_TS_LONG),
120+
Map.entry(MISSING_IP_SAMPLE_DATA.indexName, MISSING_IP_SAMPLE_DATA),
115121
Map.entry(CLIENT_IPS.indexName, CLIENT_IPS),
116122
Map.entry(CLIENT_CIDR.indexName, CLIENT_CIDR),
117123
Map.entry(AGES.indexName, AGES),

‎x-pack/plugin/esql/qa/testFixtures/src/main/resources/mapping-missing_ip_sample_data.json‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
{
2+
"properties": {
3+
"@timestamp": {
4+
"type": "date"
5+
},
6+
"event_duration": {
7+
"type": "long"
8+
},
9+
"message": {
10+
"type": "keyword"
11+
}
12+
}
13+
}

‎x-pack/plugin/esql/qa/testFixtures/src/main/resources/missing_ip_sample_data.csv‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
@timestamp:date,event_duration:long,message:keyword
2+
2023-10-23T13:55:01.543Z,1756467,Connected to 10.1.0.1
3+
2023-10-23T13:53:55.832Z,5033755,Connection error
4+
2023-10-23T13:52:55.015Z,8268153,Connection error
5+
2023-10-23T13:51:54.732Z,725448,Connection error
6+
2023-10-23T13:33:34.937Z,1232382,Disconnected
7+
2023-10-23T12:27:28.948Z,2764889,Connected to 10.1.0.2
8+
2023-10-23T12:15:03.360Z,3450233,Connected to 10.1.0.3

‎x-pack/plugin/esql/qa/testFixtures/src/main/resources/union_types.csv-spec‎

Lines changed: 68 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -405,6 +405,74 @@ count:long | message:keyword
405405
2 | Connected to 10.1.0.3
406406
;
407407

408+
multiIndexMissingIpToString
409+
required_capability: union_types
410+
required_capability: union_types_missing_field
411+
412+
FROM sample_data, sample_data_str, missing_ip_sample_data METADATA _index
413+
| EVAL client_ip = TO_STRING(client_ip)
414+
| KEEP _index, @timestamp, client_ip, event_duration, message
415+
| SORT _index ASC, @timestamp DESC
416+
;
417+
418+
_index:keyword | @timestamp:date | client_ip:keyword | event_duration:long | message:keyword
419+
missing_ip_sample_data | 2023-10-23T13:55:01.543Z | null | 1756467 | Connected to 10.1.0.1
420+
missing_ip_sample_data | 2023-10-23T13:53:55.832Z | null | 5033755 | Connection error
421+
missing_ip_sample_data | 2023-10-23T13:52:55.015Z | null | 8268153 | Connection error
422+
missing_ip_sample_data | 2023-10-23T13:51:54.732Z | null | 725448 | Connection error
423+
missing_ip_sample_data | 2023-10-23T13:33:34.937Z | null | 1232382 | Disconnected
424+
missing_ip_sample_data | 2023-10-23T12:27:28.948Z | null | 2764889 | Connected to 10.1.0.2
425+
missing_ip_sample_data | 2023-10-23T12:15:03.360Z | null | 3450233 | Connected to 10.1.0.3
426+
sample_data | 2023-10-23T13:55:01.543Z | 172.21.3.15 | 1756467 | Connected to 10.1.0.1
427+
sample_data | 2023-10-23T13:53:55.832Z | 172.21.3.15 | 5033755 | Connection error
428+
sample_data | 2023-10-23T13:52:55.015Z | 172.21.3.15 | 8268153 | Connection error
429+
sample_data | 2023-10-23T13:51:54.732Z | 172.21.3.15 | 725448 | Connection error
430+
sample_data | 2023-10-23T13:33:34.937Z | 172.21.0.5 | 1232382 | Disconnected
431+
sample_data | 2023-10-23T12:27:28.948Z | 172.21.2.113 | 2764889 | Connected to 10.1.0.2
432+
sample_data | 2023-10-23T12:15:03.360Z | 172.21.2.162 | 3450233 | Connected to 10.1.0.3
433+
sample_data_str | 2023-10-23T13:55:01.543Z | 172.21.3.15 | 1756467 | Connected to 10.1.0.1
434+
sample_data_str | 2023-10-23T13:53:55.832Z | 172.21.3.15 | 5033755 | Connection error
435+
sample_data_str | 2023-10-23T13:52:55.015Z | 172.21.3.15 | 8268153 | Connection error
436+
sample_data_str | 2023-10-23T13:51:54.732Z | 172.21.3.15 | 725448 | Connection error
437+
sample_data_str | 2023-10-23T13:33:34.937Z | 172.21.0.5 | 1232382 | Disconnected
438+
sample_data_str | 2023-10-23T12:27:28.948Z | 172.21.2.113 | 2764889 | Connected to 10.1.0.2
439+
sample_data_str | 2023-10-23T12:15:03.360Z | 172.21.2.162 | 3450233 | Connected to 10.1.0.3
440+
;
441+
442+
multiIndexMissingIpToIp
443+
required_capability: union_types
444+
required_capability: union_types_missing_field
445+
446+
FROM sample_data, sample_data_str, missing_ip_sample_data METADATA _index
447+
| EVAL client_ip = TO_IP(client_ip)
448+
| KEEP _index, @timestamp, client_ip, event_duration, message
449+
| SORT _index ASC, @timestamp DESC
450+
;
451+
452+
_index:keyword | @timestamp:date | client_ip:ip | event_duration:long | message:keyword
453+
missing_ip_sample_data | 2023-10-23T13:55:01.543Z | null | 1756467 | Connected to 10.1.0.1
454+
missing_ip_sample_data | 2023-10-23T13:53:55.832Z | null | 5033755 | Connection error
455+
missing_ip_sample_data | 2023-10-23T13:52:55.015Z | null | 8268153 | Connection error
456+
missing_ip_sample_data | 2023-10-23T13:51:54.732Z | null | 725448 | Connection error
457+
missing_ip_sample_data | 2023-10-23T13:33:34.937Z | null | 1232382 | Disconnected
458+
missing_ip_sample_data | 2023-10-23T12:27:28.948Z | null | 2764889 | Connected to 10.1.0.2
459+
missing_ip_sample_data | 2023-10-23T12:15:03.360Z | null | 3450233 | Connected to 10.1.0.3
460+
sample_data | 2023-10-23T13:55:01.543Z | 172.21.3.15 | 1756467 | Connected to 10.1.0.1
461+
sample_data | 2023-10-23T13:53:55.832Z | 172.21.3.15 | 5033755 | Connection error
462+
sample_data | 2023-10-23T13:52:55.015Z | 172.21.3.15 | 8268153 | Connection error
463+
sample_data | 2023-10-23T13:51:54.732Z | 172.21.3.15 | 725448 | Connection error
464+
sample_data | 2023-10-23T13:33:34.937Z | 172.21.0.5 | 1232382 | Disconnected
465+
sample_data | 2023-10-23T12:27:28.948Z | 172.21.2.113 | 2764889 | Connected to 10.1.0.2
466+
sample_data | 2023-10-23T12:15:03.360Z | 172.21.2.162 | 3450233 | Connected to 10.1.0.3
467+
sample_data_str | 2023-10-23T13:55:01.543Z | 172.21.3.15 | 1756467 | Connected to 10.1.0.1
468+
sample_data_str | 2023-10-23T13:53:55.832Z | 172.21.3.15 | 5033755 | Connection error
469+
sample_data_str | 2023-10-23T13:52:55.015Z | 172.21.3.15 | 8268153 | Connection error
470+
sample_data_str | 2023-10-23T13:51:54.732Z | 172.21.3.15 | 725448 | Connection error
471+
sample_data_str | 2023-10-23T13:33:34.937Z | 172.21.0.5 | 1232382 | Disconnected
472+
sample_data_str | 2023-10-23T12:27:28.948Z | 172.21.2.113 | 2764889 | Connected to 10.1.0.2
473+
sample_data_str | 2023-10-23T12:15:03.360Z | 172.21.2.162 | 3450233 | Connected to 10.1.0.3
474+
;
475+
408476
multiIndexTsLong
409477
required_capability: union_types
410478
required_capability: metadata_fields

‎x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -183,6 +183,11 @@ public enum Cap {
183183
*/
184184
UNION_TYPES_FIX_RENAME_RESOLUTION,
185185

186+
/**
187+
* Fix for union-types when some indexes are missing the required field. Done in #111932.
188+
*/
189+
UNION_TYPES_MISSING_FIELD,
190+
186191
/**
187192
* Fix a parsing issue where numbers below Long.MIN_VALUE threw an exception instead of parsing as doubles.
188193
* see <a href="https://github.com/elastic/elasticsearch/issues/104323"> Parsing large numbers is inconsistent #104323 </a>

‎x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/EsPhysicalOperationProviders.java‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -138,7 +138,9 @@ private BlockLoader getBlockLoaderFor(
138138
if (unionTypes != null) {
139139
String indexName = shardContext.ctx.index().getName();
140140
Expression conversion = unionTypes.getConversionExpressionForIndex(indexName);
141-
return new TypeConvertingBlockLoader(blockLoader, (AbstractConvertFunction) conversion);
141+
return conversion == null
142+
? BlockLoader.CONSTANT_NULLS
143+
: new TypeConvertingBlockLoader(blockLoader, (AbstractConvertFunction) conversion);
142144
}
143145
return blockLoader;
144146
}

0 commit comments

Comments
 (0)