Skip to content

Commit b170df7

Browse files
fang-xing-esqlfang-xing-esql
committed
cast counter types to the corresponding normal numeric types
1 parent f06b7a2 commit b170df7

File tree

6 files changed

+230
-2
lines changed

6 files changed

+230
-2
lines changed

‎x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/RestEsqlTestCase.java‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1318,6 +1318,8 @@ public void testTopLevelFilterBoolMerged() throws IOException {
13181318
}
13191319

13201320
public void testTopLevelFilterWithSubqueriesInFromCommand() throws IOException {
1321+
assumeTrue("subqueries in from command", EsqlCapabilities.Cap.SUBQUERY_IN_FROM_COMMAND.isEnabled());
1322+
13211323
bulkLoadTestData(10);
13221324

13231325
String query = format(null, "FROM {} , (FROM {} | WHERE integer < 8) | STATS count(*)", testIndexName(), testIndexName());

‎x-pack/plugin/esql/qa/testFixtures/src/main/resources/subquery.csv-spec‎

Lines changed: 79 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -869,3 +869,82 @@ FROM sample_data, sample_data_str,
869869
2023-10-23T13:53:55.832123456Z | 5033755 | Connection error | null | null
870870
2023-10-23T13:55:01.543123456Z | 1756467 | Connected to 10.1.0.1 | null | null
871871
;
872+
873+
subqueryInFromWithTimeSeriesDataTypesInSubquery
874+
required_capability: fork_v9
875+
required_capability: subquery_in_from_command
876+
877+
FROM sample_data, (FROM k8s metadata _index) metadata _index
878+
| WHERE @timestamp < "2024-05-10T00:01:00.000Z"
879+
| KEEP _index, @timestamp, client.ip, event_duration, cluster, network.total_bytes_in, network.eth0.tx
880+
| SORT _index, @timestamp
881+
;
882+
883+
_index:keyword | @timestamp:datetime | client.ip:ip | event_duration:long | cluster:keyword | network.total_bytes_in:long | network.eth0.tx:integer
884+
k8s | 2024-05-10T00:00:29.000Z | 10.10.20.34 | null | staging | 953 | 81
885+
k8s | 2024-05-10T00:00:33.000Z | 10.10.20.34 | null | staging | 1111 | 48
886+
k8s | 2024-05-10T00:00:51.000Z | 10.10.20.30 | null | prod | 278 | 58
887+
k8s | 2024-05-10T00:00:57.000Z | 10.10.20.30 | null | prod | 955 | 131
888+
sample_data | 2023-10-23T12:15:03.360Z | null | 3450233 | null | null | null
889+
sample_data | 2023-10-23T12:27:28.948Z | null | 2764889 | null | null | null
890+
sample_data | 2023-10-23T13:33:34.937Z | null | 1232382 | null | null | null
891+
sample_data | 2023-10-23T13:51:54.732Z | null | 725448 | null | null | null
892+
sample_data | 2023-10-23T13:52:55.015Z | null | 8268153 | null | null | null
893+
sample_data | 2023-10-23T13:53:55.832Z | null | 5033755 | null | null | null
894+
sample_data | 2023-10-23T13:55:01.543Z | null | 1756467 | null | null | null
895+
;
896+
897+
subqueryInFromWithTimeSeriesDataTypesInMainQuery
898+
required_capability: fork_v9
899+
required_capability: subquery_in_from_command
900+
901+
FROM k8s-downsampled, (FROM sample_data metadata _index) metadata _index
902+
| WHERE @timestamp <= "2024-05-09T23:30:00.000Z"
903+
| KEEP _index, @timestamp, client.ip, event_duration, cluster, network.total_bytes_in, network.eth0.tx
904+
| SORT _index, @timestamp
905+
;
906+
907+
_index:keyword | @timestamp:datetime | client.ip:ip | event_duration:long | cluster:keyword | network.total_bytes_in:long | network.eth0.tx:keyword
908+
k8s-downsampled | 2024-05-09T23:30:00.000Z | 10.10.20.30 | null | qa | 1143 | null
909+
k8s-downsampled | 2024-05-09T23:30:00.000Z | 10.10.20.31 | null | qa | 363 | null
910+
k8s-downsampled | 2024-05-09T23:30:00.000Z | 10.10.20.33 | null | prod | 210 | null
911+
k8s-downsampled | 2024-05-09T23:30:00.000Z | 10.10.20.30 | null | prod | 285 | null
912+
k8s-downsampled | 2024-05-09T23:30:00.000Z | 10.10.20.31 | null | prod | 1038 | null
913+
k8s-downsampled | 2024-05-09T23:30:00.000Z | 10.10.20.31 | null | qa | 1032 | null
914+
k8s-downsampled | 2024-05-09T23:30:00.000Z | 10.10.20.34 | null | staging | 821 | null
915+
k8s-downsampled | 2024-05-09T23:30:00.000Z | 10.10.20.34 | null | staging | 838 | null
916+
k8s-downsampled | 2024-05-09T23:30:00.000Z | 10.10.20.30 | null | staging | 930 | null
917+
sample_data | 2023-10-23T12:15:03.360Z | null | 3450233 | null | null | null
918+
sample_data | 2023-10-23T12:27:28.948Z | null | 2764889 | null | null | null
919+
sample_data | 2023-10-23T13:33:34.937Z | null | 1232382 | null | null | null
920+
sample_data | 2023-10-23T13:51:54.732Z | null | 725448 | null | null | null
921+
sample_data | 2023-10-23T13:52:55.015Z | null | 8268153 | null | null | null
922+
sample_data | 2023-10-23T13:53:55.832Z | null | 5033755 | null | null | null
923+
sample_data | 2023-10-23T13:55:01.543Z | null | 1756467 | null | null | null
924+
;
925+
926+
subqueryInFromWithTimeSeriesDataTypesInMainQueryAndSubquery
927+
required_capability: fork_v9
928+
required_capability: subquery_in_from_command
929+
930+
FROM k8s, (FROM k8s-downsampled metadata _index | WHERE @timestamp <= "2024-05-09T23:30:00.000Z") metadata _index
931+
| WHERE @timestamp <= "2024-05-10T00:01:00.000Z"
932+
| KEEP _index, @timestamp, client.ip, cluster, network.total_bytes_in, network.eth0.tx
933+
| SORT _index, @timestamp
934+
;
935+
936+
_index:keyword | @timestamp:datetime | client.ip:ip | cluster:keyword | network.total_bytes_in:long | network.eth0.tx:keyword
937+
k8s | 2024-05-10T00:00:29.000Z | 10.10.20.34 | staging | 953 | null
938+
k8s | 2024-05-10T00:00:33.000Z | 10.10.20.34 | staging | 1111 | null
939+
k8s | 2024-05-10T00:00:51.000Z | 10.10.20.30 | prod | 278 | null
940+
k8s | 2024-05-10T00:00:57.000Z | 10.10.20.30 | prod | 955 | null
941+
k8s-downsampled | 2024-05-09T23:30:00.000Z | 10.10.20.31 | prod | 1038 | null
942+
k8s-downsampled | 2024-05-09T23:30:00.000Z | 10.10.20.30 | staging | 930 | null
943+
k8s-downsampled | 2024-05-09T23:30:00.000Z | 10.10.20.34 | staging | 838 | null
944+
k8s-downsampled | 2024-05-09T23:30:00.000Z | 10.10.20.30 | qa | 1143 | null
945+
k8s-downsampled | 2024-05-09T23:30:00.000Z | 10.10.20.31 | qa | 1032 | null
946+
k8s-downsampled | 2024-05-09T23:30:00.000Z | 10.10.20.30 | prod | 285 | null
947+
k8s-downsampled | 2024-05-09T23:30:00.000Z | 10.10.20.31 | qa | 363 | null
948+
k8s-downsampled | 2024-05-09T23:30:00.000Z | 10.10.20.33 | prod | 210 | null
949+
k8s-downsampled | 2024-05-09T23:30:00.000Z | 10.10.20.34 | staging | 821 | null
950+
;

‎x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -862,6 +862,9 @@ private LogicalPlan resolveFork(Fork fork, AnalyzerContext context) {
862862
// We cannot assign an alias with an UNSUPPORTED data type, so we use another type that is
863863
// supported. This way we can add this missing column containing only null values to the fork branch output.
864864
var attrType = attr.dataType() == UNSUPPORTED ? KEYWORD : attr.dataType();
865+
if (attrType.isCounter()) {
866+
attrType = attrType.noCounter();
867+
}
865868
return new Alias(source, attr.name(), new Literal(attr.source(), null, attrType));
866869
}).toList();
867870

@@ -2485,6 +2488,15 @@ private DataType commonType(DataType t1, DataType t2) {
24852488
if (t1.isDate() && t2.isDate() && t1 != t2) {
24862489
return DATE_NANOS;
24872490
}
2491+
2492+
if (t1.isCounter()) {
2493+
t1 = t1.noCounter();
2494+
}
2495+
2496+
if (t2.isCounter()) {
2497+
t2 = t2.noCounter();
2498+
}
2499+
24882500
return EsqlDataTypeConverter.commonType(t1, t2);
24892501
}
24902502

‎x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/EsqlSession.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -531,7 +531,7 @@ private void preAnalyzeSubqueryIndex(
531531
);
532532
} else {
533533
// occurs when dealing with local relations (row a = 1)
534-
listener.onResponse(result.withIndices(IndexResolution.invalid("[none specified]")));
534+
listener.onResponse(result.addSubqueryIndexResolution("invalid subquery", IndexResolution.invalid("[none specified]")));
535535
}
536536
}
537537

‎x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTestUtils.java‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -230,7 +230,9 @@ public static Map<String, IndexResolution> defaultSubqueryResolution() {
230230
"sample_data",
231231
loadMapping("mapping-sample_data.json", "sample_data"),
232232
"test_mixed_types",
233-
loadMapping("mapping-default-incompatible.json", "test_mixed_types")
233+
loadMapping("mapping-default-incompatible.json", "test_mixed_types"),
234+
"k8s",
235+
loadMapping("k8s-downsampled-mappings.json", "k8s", IndexMode.TIME_SERIES)
234236
);
235237
}
236238

‎x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java‎

Lines changed: 133 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -84,6 +84,7 @@
8484
import org.elasticsearch.xpack.esql.plan.logical.Eval;
8585
import org.elasticsearch.xpack.esql.plan.logical.Filter;
8686
import org.elasticsearch.xpack.esql.plan.logical.Fork;
87+
import org.elasticsearch.xpack.esql.plan.logical.InlineStats;
8788
import org.elasticsearch.xpack.esql.plan.logical.Insist;
8889
import org.elasticsearch.xpack.esql.plan.logical.Limit;
8990
import org.elasticsearch.xpack.esql.plan.logical.LogicalPlan;
@@ -4989,6 +4990,138 @@ public void testMixedDataTypesWithExplicitCastingInSubquery() {
49894990
assertEquals("test_mixed_types", subqueryIndex.indexPattern());
49904991
}
49914992

4993+
public void testSubqueryWithTimeSeriesIndexInMainQuery() {
4994+
assumeTrue("Requires subquery in FROM command support", EsqlCapabilities.Cap.SUBQUERY_IN_FROM_COMMAND.isEnabled());
4995+
LogicalPlan plan = analyze("""
4996+
FROM k8s, (FROM sample_data), (FROM sample_data | WHERE client_ip == "127.0.0.1")
4997+
| WHERE @timestamp > "2025-10-07"
4998+
""", "k8s-downsampled-mappings.json");
4999+
5000+
Limit limit = as(plan, Limit.class);
5001+
Filter filter = as(limit.child(), Filter.class);
5002+
UnionAll unionAll = as(filter.child(), UnionAll.class);
5003+
List<Attribute> output = unionAll.output();
5004+
// all fields from the three indices
5005+
assertEquals(24, output.size());
5006+
assertEquals(3, unionAll.children().size());
5007+
5008+
limit = as(unionAll.children().get(0), Limit.class);
5009+
EsqlProject esqlProject = as(limit.child(), EsqlProject.class);
5010+
Eval eval = as(esqlProject.child(), Eval.class);
5011+
eval = as(eval.child(), Eval.class);
5012+
EsRelation relation = as(eval.child(), EsRelation.class);
5013+
assertEquals("k8s", relation.indexPattern());
5014+
assertEquals(IndexMode.STANDARD, relation.indexMode());
5015+
5016+
limit = as(unionAll.children().get(1), Limit.class);
5017+
esqlProject = as(limit.child(), EsqlProject.class);
5018+
eval = as(esqlProject.child(), Eval.class);
5019+
Subquery subquery = as(eval.child(), Subquery.class);
5020+
relation = as(subquery.child(), EsRelation.class);
5021+
assertEquals("sample_data", relation.indexPattern());
5022+
assertEquals(IndexMode.STANDARD, relation.indexMode());
5023+
5024+
limit = as(unionAll.children().get(2), Limit.class);
5025+
esqlProject = as(limit.child(), EsqlProject.class);
5026+
eval = as(esqlProject.child(), Eval.class);
5027+
subquery = as(eval.child(), Subquery.class);
5028+
filter = as(subquery.child(), Filter.class);
5029+
relation = as(filter.child(), EsRelation.class);
5030+
assertEquals("sample_data", relation.indexPattern());
5031+
assertEquals(IndexMode.STANDARD, relation.indexMode());
5032+
}
5033+
5034+
public void testSubqueryWithTimeSeriesIndexInSubquery() {
5035+
assumeTrue("Requires subquery in FROM command support", EsqlCapabilities.Cap.SUBQUERY_IN_FROM_COMMAND.isEnabled());
5036+
LogicalPlan plan = analyze("""
5037+
FROM sample_data,
5038+
(FROM k8s | EVAL a = TO_AGGREGATE_METRIC_DOUBLE(1) | INLINE STATS tx_max = MAX(network.eth0.tx) BY pod),
5039+
(FROM sample_data | WHERE client_ip == "127.0.0.1")
5040+
| WHERE @timestamp > "2025-10-07"
5041+
""", "mapping-sample_data.json");
5042+
5043+
Limit limit = as(plan, Limit.class);
5044+
Filter filter = as(limit.child(), Filter.class);
5045+
UnionAll unionAll = as(filter.child(), UnionAll.class);
5046+
List<Attribute> output = unionAll.output();
5047+
assertEquals(26, output.size());
5048+
assertEquals(3, unionAll.children().size());
5049+
5050+
limit = as(unionAll.children().get(0), Limit.class);
5051+
EsqlProject esqlProject = as(limit.child(), EsqlProject.class);
5052+
Eval eval = as(esqlProject.child(), Eval.class);
5053+
EsRelation relation = as(eval.child(), EsRelation.class);
5054+
assertEquals("sample_data", relation.indexPattern());
5055+
assertEquals(IndexMode.STANDARD, relation.indexMode());
5056+
5057+
limit = as(unionAll.children().get(1), Limit.class);
5058+
esqlProject = as(limit.child(), EsqlProject.class);
5059+
eval = as(esqlProject.child(), Eval.class);
5060+
eval = as(eval.child(), Eval.class);
5061+
Subquery subquery = as(eval.child(), Subquery.class);
5062+
InlineStats inlineStats = as(subquery.child(), InlineStats.class);
5063+
Aggregate aggregate = as(inlineStats.child(), Aggregate.class);
5064+
eval = as(aggregate.child(), Eval.class);
5065+
relation = as(eval.child(), EsRelation.class);
5066+
assertEquals("k8s", relation.indexPattern());
5067+
assertEquals(IndexMode.STANDARD, relation.indexMode());
5068+
5069+
limit = as(unionAll.children().get(2), Limit.class);
5070+
esqlProject = as(limit.child(), EsqlProject.class);
5071+
eval = as(esqlProject.child(), Eval.class);
5072+
subquery = as(eval.child(), Subquery.class);
5073+
filter = as(subquery.child(), Filter.class);
5074+
relation = as(filter.child(), EsRelation.class);
5075+
assertEquals("sample_data", relation.indexPattern());
5076+
assertEquals(IndexMode.STANDARD, relation.indexMode());
5077+
}
5078+
5079+
public void testSubqueryWithTimeSeriesIndexInMainQueryAndSubquery() {
5080+
assumeTrue("Requires subquery in FROM command support", EsqlCapabilities.Cap.SUBQUERY_IN_FROM_COMMAND.isEnabled());
5081+
LogicalPlan plan = analyze("""
5082+
FROM k8s,
5083+
(FROM k8s | EVAL a = TO_AGGREGATE_METRIC_DOUBLE(1) | INLINE STATS tx_max = MAX(network.eth0.tx) BY pod),
5084+
(FROM sample_data | WHERE client_ip == "127.0.0.1")
5085+
| WHERE @timestamp > "2025-10-07"
5086+
""", "k8s-downsampled-mappings.json");
5087+
5088+
Limit limit = as(plan, Limit.class);
5089+
Filter filter = as(limit.child(), Filter.class);
5090+
UnionAll unionAll = as(filter.child(), UnionAll.class);
5091+
List<Attribute> output = unionAll.output();
5092+
assertEquals(26, output.size());
5093+
assertEquals(3, unionAll.children().size());
5094+
5095+
limit = as(unionAll.children().get(0), Limit.class);
5096+
EsqlProject esqlProject = as(limit.child(), EsqlProject.class);
5097+
Eval eval = as(esqlProject.child(), Eval.class);
5098+
eval = as(eval.child(), Eval.class);
5099+
EsRelation relation = as(eval.child(), EsRelation.class);
5100+
assertEquals("k8s", relation.indexPattern());
5101+
assertEquals(IndexMode.STANDARD, relation.indexMode());
5102+
5103+
limit = as(unionAll.children().get(1), Limit.class);
5104+
esqlProject = as(limit.child(), EsqlProject.class);
5105+
eval = as(esqlProject.child(), Eval.class);
5106+
eval = as(eval.child(), Eval.class);
5107+
Subquery subquery = as(eval.child(), Subquery.class);
5108+
InlineStats inlineStats = as(subquery.child(), InlineStats.class);
5109+
Aggregate aggregate = as(inlineStats.child(), Aggregate.class);
5110+
eval = as(aggregate.child(), Eval.class);
5111+
relation = as(eval.child(), EsRelation.class);
5112+
assertEquals("k8s", relation.indexPattern());
5113+
assertEquals(IndexMode.STANDARD, relation.indexMode());
5114+
5115+
limit = as(unionAll.children().get(2), Limit.class);
5116+
esqlProject = as(limit.child(), EsqlProject.class);
5117+
eval = as(esqlProject.child(), Eval.class);
5118+
subquery = as(eval.child(), Subquery.class);
5119+
filter = as(subquery.child(), Filter.class);
5120+
relation = as(filter.child(), EsRelation.class);
5121+
assertEquals("sample_data", relation.indexPattern());
5122+
assertEquals(IndexMode.STANDARD, relation.indexMode());
5123+
}
5124+
49925125
private void verifyNameAndType(String actualName, DataType actualType, String expectedName, DataType expectedType) {
49935126
assertEquals(expectedName, actualName);
49945127
assertEquals(expectedType, actualType);

0 commit comments

Comments
 (0)