From 9143fef7a6f99ce62a4ddd998eaa3bb909925961 Mon Sep 17 00:00:00 2001 From: Andrei Stefan Date: Tue, 19 Aug 2025 17:43:34 +0300 Subject: [PATCH 01/11] Tests for inlinestats with union types --- .../xpack/esql/ccq/MultiClusterSpecIT.java | 4 +- .../src/main/resources/inlinestats.csv-spec | 1260 +++++++++++++++-- .../src/main/resources/union_types.csv-spec | 68 - .../xpack/esql/action/EsqlCapabilities.java | 2 +- .../xpack/esql/analysis/AnalyzerTests.java | 2 +- .../logical/PropagateInlineEvalsTests.java | 4 +- .../esql/session/FieldNameUtilsTests.java | 36 +- 7 files changed, 1205 insertions(+), 171 deletions(-) diff --git a/x-pack/plugin/esql/qa/server/multi-clusters/src/javaRestTest/java/org/elasticsearch/xpack/esql/ccq/MultiClusterSpecIT.java b/x-pack/plugin/esql/qa/server/multi-clusters/src/javaRestTest/java/org/elasticsearch/xpack/esql/ccq/MultiClusterSpecIT.java index b140f7ded44f0..4b90aa1a69c80 100644 --- a/x-pack/plugin/esql/qa/server/multi-clusters/src/javaRestTest/java/org/elasticsearch/xpack/esql/ccq/MultiClusterSpecIT.java +++ b/x-pack/plugin/esql/qa/server/multi-clusters/src/javaRestTest/java/org/elasticsearch/xpack/esql/ccq/MultiClusterSpecIT.java @@ -51,7 +51,7 @@ import static org.elasticsearch.xpack.esql.action.EsqlCapabilities.Cap.ENABLE_LOOKUP_JOIN_ON_REMOTE; import static org.elasticsearch.xpack.esql.action.EsqlCapabilities.Cap.FORK_V9; import static org.elasticsearch.xpack.esql.action.EsqlCapabilities.Cap.INLINESTATS; -import static org.elasticsearch.xpack.esql.action.EsqlCapabilities.Cap.INLINESTATS_V9; +import static org.elasticsearch.xpack.esql.action.EsqlCapabilities.Cap.INLINESTATS_V11; import static org.elasticsearch.xpack.esql.action.EsqlCapabilities.Cap.JOIN_LOOKUP_V12; import static org.elasticsearch.xpack.esql.action.EsqlCapabilities.Cap.JOIN_PLANNING_V1; import static org.elasticsearch.xpack.esql.action.EsqlCapabilities.Cap.METADATA_FIELDS_REMOTE_TEST; @@ -137,7 +137,7 @@ protected void shouldSkipTest(String testName) throws IOException { assumeTrue("Test " + testName + " is skipped on " + oldVersion, isEnabled(testName, instructions, oldVersion)); assumeFalse("INLINESTATS not yet supported in CCS", testCase.requiredCapabilities.contains(INLINESTATS.capabilityName())); assumeFalse("INLINESTATS not yet supported in CCS", testCase.requiredCapabilities.contains(JOIN_PLANNING_V1.capabilityName())); - assumeFalse("INLINESTATS not yet supported in CCS", testCase.requiredCapabilities.contains(INLINESTATS_V9.capabilityName())); + assumeFalse("INLINESTATS not yet supported in CCS", testCase.requiredCapabilities.contains(INLINESTATS_V11.capabilityName())); if (testCase.requiredCapabilities.contains(JOIN_LOOKUP_V12.capabilityName())) { assumeTrue( "LOOKUP JOIN not yet supported in CCS", diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/inlinestats.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/inlinestats.csv-spec index b90135f43a417..943de99f7d02a 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/inlinestats.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/inlinestats.csv-spec @@ -3,7 +3,7 @@ // allFieldsReturned -required_capability: inlinestats_v9 +required_capability: inlinestats_v11 FROM hosts METADATA _index | INLINESTATS c = COUNT(*) BY host_group @@ -16,7 +16,7 @@ eth0 |epsilon gw instance|epsilon |[fe80::cae2:65ff:fece:feb9, ; maxOfInt -required_capability: inlinestats_v9 +required_capability: inlinestats_v11 // tag::max-languages[] FROM employees | KEEP emp_no, languages @@ -38,7 +38,7 @@ emp_no:integer | languages:integer | max_lang:integer ; maxOfIntByKeyword -required_capability: inlinestats_v9 +required_capability: inlinestats_v11 FROM employees | KEEP emp_no, languages, gender @@ -56,7 +56,7 @@ emp_no:integer | languages:integer | max_lang:integer | gender:keyword ; maxOfLongByKeyword -required_capability: inlinestats_v9 +required_capability: inlinestats_v11 FROM employees | KEEP emp_no, avg_worked_seconds, gender @@ -71,7 +71,7 @@ emp_no:integer | avg_worked_seconds:long | max_avg_worked_seconds:long | gender: ; maxOfLong -required_capability: inlinestats_v9 +required_capability: inlinestats_v11 FROM employees | KEEP emp_no, avg_worked_seconds, gender @@ -84,7 +84,7 @@ emp_no:integer | avg_worked_seconds:long | gender:keyword | max_avg_worked_secon ; maxOfLongByCalculatedKeyword -required_capability: inlinestats_v9 +required_capability: inlinestats_v11 // tag::longest-tenured-by-first[] FROM employees @@ -107,7 +107,7 @@ emp_no:integer | avg_worked_seconds:long | last_name:keyword | max_avg_worked_se ; maxOfLongByCalculatedNamedKeyword -required_capability: inlinestats_v9 +required_capability: inlinestats_v11 FROM employees | KEEP emp_no, avg_worked_seconds, last_name @@ -126,7 +126,7 @@ emp_no:integer | avg_worked_seconds:long | last_name:keyword | max_avg_worked_se ; maxOfLongByCalculatedDroppedKeyword -required_capability: inlinestats_v9 +required_capability: inlinestats_v11 FROM employees | INLINESTATS max_avg_worked_seconds = MAX(avg_worked_seconds) BY l = SUBSTRING(last_name, 0, 1) @@ -145,7 +145,7 @@ emp_no:integer | avg_worked_seconds:long | last_name:keyword | max_avg_worked_se ; maxOfLongByEvaledKeyword -required_capability: inlinestats_v9 +required_capability: inlinestats_v11 FROM employees | EVAL l = SUBSTRING(last_name, 0, 1) @@ -165,7 +165,7 @@ emp_no:integer | avg_worked_seconds:long | max_avg_worked_seconds:long | l:keywo ; maxOfLongByInt -required_capability: inlinestats_v9 +required_capability: inlinestats_v11 FROM employees | KEEP emp_no, avg_worked_seconds, languages @@ -183,7 +183,7 @@ emp_no:integer | avg_worked_seconds:long | max_avg_worked_seconds:long | languag ; maxOfLongByIntDouble -required_capability: inlinestats_v9 +required_capability: inlinestats_v11 FROM employees | KEEP emp_no, avg_worked_seconds, languages, height @@ -201,7 +201,7 @@ emp_no:integer | avg_worked_seconds:long | max_avg_worked_seconds:long | languag ; two -required_capability: inlinestats_v9 +required_capability: inlinestats_v11 FROM employees | KEEP emp_no, languages, avg_worked_seconds, gender @@ -225,7 +225,7 @@ emp_no:integer |avg_worked_seconds:long|avg_avg_worked_seconds:double|languages: ; three -required_capability: inlinestats_v9 +required_capability: inlinestats_v11 // used to fail with AssertionError at org.elasticsearch.xpack.esql.plan.logical.Limit.writeTo(Limit.java:70) FROM employees @@ -253,7 +253,7 @@ emp_no:integer |avg_worked_seconds:long|avg_avg_worked_seconds:double|languages: // TODO: INLINESTATS unit test needed for this one pushDownSort_To_LeftSideOnly -required_capability: inlinestats_v9 +required_capability: inlinestats_v11 from employees | sort emp_no @@ -271,7 +271,7 @@ from employees ; byMultivaluedSimple -required_capability: inlinestats_v9 +required_capability: inlinestats_v11 // tag::mv-group[] FROM airports @@ -289,7 +289,7 @@ abbrev:keyword | type:keyword | scalerank:integer | min_scalerank:integer ; byMultivaluedMvExpand -required_capability: inlinestats_v9 +required_capability: inlinestats_v11 // tag::mv-expand[] FROM airports @@ -309,7 +309,7 @@ GWL |9 |4 |military ; byMvExpand -required_capability: inlinestats_v9 +required_capability: inlinestats_v11 // tag::extreme-airports[] FROM airports @@ -338,7 +338,7 @@ FROM airports ; mvMinMvExpand -required_capability: inlinestats_v9 +required_capability: inlinestats_v11 FROM airports | EVAL original_type = type @@ -361,7 +361,7 @@ ZAR |Zaria |POINT (7.7 11.0667) |Nigeria |POINT ( ; afterStats -required_capability: inlinestats_v9 +required_capability: inlinestats_v11 FROM airports | STATS count=COUNT(*) BY country @@ -384,7 +384,7 @@ count:long | country:keyword | avg:double ; afterWhere -required_capability: inlinestats_v9 +required_capability: inlinestats_v11 FROM airports | WHERE country != "United States" @@ -402,7 +402,7 @@ abbrev:keyword | country:keyword | count:long ; afterLookup -required_capability: inlinestats_v9 +required_capability: inlinestats_v11 required_capability: join_lookup_v12 FROM airports @@ -426,7 +426,7 @@ ZNZ |4 |German ; afterEnrich -required_capability: inlinestats_v9 +required_capability: inlinestats_v11 required_capability: enrich_load FROM airports @@ -447,7 +447,7 @@ abbrev:keyword | city:keyword | "COUNT(*)":long | region:text ; beforeStats -required_capability: inlinestats_v9 +required_capability: inlinestats_v11 FROM airports | EVAL lat = ST_Y(location) @@ -460,7 +460,7 @@ northern:long | southern:long ; beforeKeepSort -required_capability: inlinestats_v9 +required_capability: inlinestats_v11 FROM employees | INLINESTATS max_salary = MAX(salary) by languages @@ -475,7 +475,7 @@ emp_no:integer | languages:integer | max_salary:integer ; beforeKeepWhere -required_capability: inlinestats_v9 +required_capability: inlinestats_v11 FROM employees | INLINESTATS max_salary = MAX(salary) by languages @@ -488,7 +488,7 @@ emp_no:integer | languages:integer | max_salary:integer ; beforeEnrich -required_capability: inlinestats_v9 +required_capability: inlinestats_v11 required_capability: enrich_load FROM airports @@ -507,7 +507,7 @@ ACA |Acapulco de Juárez|385 |major |Acapulco de ; beforeAndAfterEnrich -required_capability: inlinestats_v9 +required_capability: inlinestats_v11 required_capability: enrich_load FROM airports @@ -530,7 +530,7 @@ ALL |Albenga |499 |mid |1 ; shadowing -required_capability: inlinestats_v9 +required_capability: inlinestats_v11 ROW left = "left", client_ip = "172.21.0.5", env = "env", right = "right" | INLINESTATS env = VALUES(right) BY client_ip @@ -541,7 +541,7 @@ left | right | right | 172.21.0.5 ; shadowingMulti -required_capability: inlinestats_v9 +required_capability: inlinestats_v11 ROW left = "left", airport = "Zurich Airport ZRH", city = "Zürich", middle = "middle", region = "North-East Switzerland", right = "right" | INLINESTATS airport=VALUES(left), region=VALUES(left), city_boundary=VALUES(left) BY city @@ -552,7 +552,7 @@ left | middle | right | left | left ; shadowingSelf -required_capability: inlinestats_v9 +required_capability: inlinestats_v11 ROW city = "Raleigh" | INLINESTATS city = COUNT(city) @@ -563,7 +563,7 @@ city:long ; shadowingSelfBySelf -required_capability: inlinestats_v9 +required_capability: inlinestats_v11 ROW city = "Raleigh" | INLINESTATS city = COUNT(city) BY city @@ -575,7 +575,7 @@ Raleigh ; shadowingInternal -required_capability: inlinestats_v9 +required_capability: inlinestats_v11 ROW city = "Zürich" | INLINESTATS x = VALUES(city), x = VALUES(city) @@ -587,7 +587,7 @@ Zürich | Zürich ; multiInlinestatsWithRow -required_capability: inlinestats_v9 +required_capability: inlinestats_v11 row x = 1 | inlinestats x = max(x) + min(x) @@ -601,7 +601,7 @@ row x = 1 ; ignoreUnusedEvaledValue_AndInlineStats -required_capability: inlinestats_v9 +required_capability: inlinestats_v11 ROW x = 1 | INLINESTATS max(x) @@ -614,7 +614,7 @@ x:integer ; ignoreUnusedEvaledValue_AndInlineStats2 -required_capability: inlinestats_v9 +required_capability: inlinestats_v11 ROW x = 1, z = 2 | INLINESTATS max(x) @@ -627,7 +627,7 @@ x:integer | z:integer ; ignoreUnusedEvaledValue_AndInlineStats3 -required_capability: inlinestats_v9 +required_capability: inlinestats_v11 from employees | inlinestats max(salary) @@ -642,7 +642,7 @@ from employees ; ignoreUnusedEvaledValue_AndInlineStats4 -required_capability: inlinestats_v9 +required_capability: inlinestats_v11 from employees | inlinestats max(salary), m = min(salary) by gender @@ -657,7 +657,7 @@ emp_no:integer ; ignoreUnusedEvaledValue_AndInlineStats5 -required_capability: inlinestats_v9 +required_capability: inlinestats_v11 from employees | inlinestats max(salary), m = min(salary) by gender @@ -672,7 +672,7 @@ emp_no:integer ; shadowEntireInlinestats -required_capability: inlinestats_v9 +required_capability: inlinestats_v11 FROM employees | INLINESTATS x = avg(salary), y = min(salary) BY emp_no @@ -687,7 +687,7 @@ x:integer |y:integer |emp_no:integer ; byConstant -required_capability: inlinestats_v9 +required_capability: inlinestats_v11 FROM employees | KEEP emp_no, languages @@ -706,7 +706,7 @@ emp_no:integer | languages:integer | max_lang:integer | y:integer ; aggConstant -required_capability: inlinestats_v9 +required_capability: inlinestats_v11 FROM employees | KEEP emp_no @@ -724,7 +724,7 @@ one:integer | emp_no:integer ; percentile -required_capability: inlinestats_v9 +required_capability: inlinestats_v11 FROM employees | KEEP emp_no, salary @@ -743,7 +743,7 @@ emp_no:integer | salary:integer | ninety_fifth_salary:double ; byTwoCalculated -required_capability: inlinestats_v9 +required_capability: inlinestats_v11 FROM airports | WHERE abbrev IS NOT NULL @@ -763,7 +763,7 @@ abbrev:keyword | scalerank:integer | location:geo_point byTwoCalculatedSecondOverwrites required_capability: stats_alias_collision_warnings -required_capability: inlinestats_v9 +required_capability: inlinestats_v11 FROM airports | WHERE abbrev IS NOT NULL @@ -784,7 +784,7 @@ abbrev:keyword | scalerank:integer | location:geo_point byTwoCalculatedSecondOverwritesReferencingFirst required_capability: stats_alias_collision_warnings -required_capability: inlinestats_v9 +required_capability: inlinestats_v11 FROM airports | WHERE abbrev IS NOT NULL @@ -807,7 +807,7 @@ abbrev:keyword | scalerank:integer | location:geo_point groupShadowsAgg required_capability: stats_alias_collision_warnings -required_capability: inlinestats_v9 +required_capability: inlinestats_v11 FROM airports | WHERE abbrev IS NOT NULL @@ -827,7 +827,7 @@ abbrev:keyword | scalerank:integer | location:geo_point ; groupShadowsField -required_capability: inlinestats_v9 +required_capability: inlinestats_v11 FROM employees | KEEP emp_no, salary, hire_date @@ -846,7 +846,7 @@ emp_no:integer | salary:integer | avg_salary:double | hire_date:datetime ; groupByExpression_And_ExistentField -required_capability: inlinestats_v9 +required_capability: inlinestats_v11 FROM employees | KEEP emp_no, languages, gender | EVAL x = "ABC" @@ -864,7 +864,7 @@ emp_no:integer | languages:integer | x:keyword | max_lang:integer | y:keyword | ; groupByRenamedColumn -required_capability: inlinestats_v9 +required_capability: inlinestats_v11 FROM employees | KEEP emp_no, languages, gender | INLINESTATS max_lang = MAX(languages) BY y = gender @@ -883,7 +883,7 @@ emp_no:integer | languages:integer | gender:keyword | max_lang:integer | y:keywo // fails with AssertionError at org.elasticsearch.xpack.esql.plan.logical.Limit.writeTo(Limit.java:70) groupByMultipleRenamedColumns_AndOneExpression_Last -required_capability: inlinestats_v9 +required_capability: inlinestats_v11 FROM employees | KEEP emp_no, languages, gender, first_name @@ -907,7 +907,7 @@ emp_no:integer | languages:integer | gender:keyword|first_name:keyword|max_lang: // fails with AssertionError at org.elasticsearch.xpack.esql.plan.logical.Limit.writeTo(Limit.java:70) groupByMultipleRenamedColumns_AndTwoExpressions -required_capability: inlinestats_v9 +required_capability: inlinestats_v11 FROM employees | KEEP emp_no, languages, gender, first_name @@ -931,7 +931,7 @@ emp_no:integer | languages:integer | gender:keyword|first_name:keyword|max_lang: // fails with AssertionError at org.elasticsearch.xpack.esql.plan.logical.Limit.writeTo(Limit.java:70) groupByMultipleRenamedColumns_AndMultipleRenames -required_capability: inlinestats_v9 +required_capability: inlinestats_v11 FROM employees | KEEP emp_no, languages, gender, first_name @@ -956,7 +956,7 @@ emp_no:integer | languages:integer | gender:keyword| f:keyword |max_lang: // fails with AssertionError at org.elasticsearch.xpack.esql.plan.logical.Limit.writeTo(Limit.java:70) groupByMultipleRenamedColumns_AndSameNameExpressionGroupingOverride -required_capability: inlinestats_v9 +required_capability: inlinestats_v11 FROM employees | KEEP emp_no, languages, gender, first_name @@ -980,7 +980,7 @@ emp_no:integer | languages:integer | gender:keyword|max_lang:integer| y:keyword ; twoAggregatesGroupedBy_AField_And_AnExpression -required_capability: inlinestats_v9 +required_capability: inlinestats_v11 FROM employees | KEEP emp_no, languages, gender, last_name @@ -1002,7 +1002,7 @@ emp_no:integer |languages:integer|last_name:keyword|max_lang:integer|min_lang:in ; groupByMultipleRenamedColumns_InversedOrder -required_capability: inlinestats_v9 +required_capability: inlinestats_v11 FROM employees | KEEP emp_no, languages, still_hired, gender @@ -1020,7 +1020,7 @@ emp_no:integer |languages:integer|still_hired:boolean| gender:keyword|max_lang:i ; groupByMultipleRenamedColumns_InversedOrder_ComplexEval -required_capability: inlinestats_v9 +required_capability: inlinestats_v11 FROM employees | KEEP emp_no, languages, still_hired, gender @@ -1039,7 +1039,7 @@ emp_no:integer |languages:integer|still_hired:boolean| gender:keyword|multilingu ; groupByMultipleRenamedColumns_AndComplexEval -required_capability: inlinestats_v9 +required_capability: inlinestats_v11 FROM employees | KEEP emp_no, languages, still_hired, gender @@ -1059,7 +1059,7 @@ emp_no:integer |languages:integer|still_hired:boolean| gender:keyword|multilingu // fails with AssertionError at org.elasticsearch.xpack.esql.plan.logical.Limit.writeTo(Limit.java:70) groupByMultipleRenamedColumns_AndConstantValue -required_capability: inlinestats_v9 +required_capability: inlinestats_v11 FROM employees | KEEP emp_no, languages, gender, first_name @@ -1083,7 +1083,7 @@ emp_no:integer |languages:integer|gender:keyword |first_name:keyword | x:keyw ; groupByRenamedExpression -required_capability: inlinestats_v9 +required_capability: inlinestats_v11 FROM employees | KEEP emp_no, languages, gender, last_name @@ -1105,7 +1105,7 @@ emp_no:integer |languages:integer|last_name:keyword|max_lang:integer|min_lang:in ; doubleFilterOnLeftAndRight_InlineStats_Sides -required_capability: inlinestats_v9 +required_capability: inlinestats_v11 FROM employees | INLINESTATS max_salary = MAX(salary), min_salary = MIN(salary) by languages @@ -1126,7 +1126,7 @@ emp_no:integer |languages:integer|salary:integer |max_salary:integer|min_salary: ; filterOnInlineStatsAggs -required_capability: inlinestats_v9 +required_capability: inlinestats_v11 FROM employees | INLINESTATS max_salary = MAX(salary), min_salary = MIN(salary) by languages @@ -1145,7 +1145,7 @@ emp_no:integer |languages:integer|salary:integer |max_salary:integer|min_salary: ; filterOnInlineStatsAggsValues_And_Groupings -required_capability: inlinestats_v9 +required_capability: inlinestats_v11 FROM employees | INLINESTATS max_salary = MAX(salary), min_salary = MIN(salary) by languages @@ -1164,7 +1164,7 @@ emp_no:integer |languages:integer|salary:integer |max_salary:integer|min_salary: ; inlineStatsOverrideEVALed_FieldWithSameName -required_capability: inlinestats_v9 +required_capability: inlinestats_v11 FROM hosts METADATA _index | EVAL x = ip1 @@ -1178,7 +1178,7 @@ beta k8s server |beta |127.0.0.1 |hosts |127.0.0.2|2 ; doubleShadowing -required_capability: inlinestats_v9 +required_capability: inlinestats_v11 FROM employees | INLINESTATS salary = min(salary) BY gender @@ -1197,7 +1197,7 @@ salary:integer |gender:keyword ; doubleShadowing_WithIntertwinedFilters -required_capability: inlinestats_v9 +required_capability: inlinestats_v11 FROM employees | WHERE salary > 30000 @@ -1222,7 +1222,7 @@ salary:integer |gender:keyword ; shadowingAggregateByNextGrouping -required_capability: inlinestats_v9 +required_capability: inlinestats_v11 FROM employees | KEEP gender, languages, emp_no, salary @@ -1239,7 +1239,7 @@ emp_no:integer |salary:integer |languages:integer|avg(salary):double|gender:long ; doubleShadowingWithEval -required_capability: inlinestats_v9 +required_capability: inlinestats_v11 from employees | eval salary = salary/100 @@ -1259,7 +1259,7 @@ salary:integer|gender:keyword ; doubleShadowingWithDoubleStats -required_capability: inlinestats_v9 +required_capability: inlinestats_v11 from employees | stats salary=min(salary) by gender @@ -1276,7 +1276,7 @@ M |25324 ; renamingGroupingWithItself -required_capability: inlinestats_v9 +required_capability: inlinestats_v11 FROM employees | EVAL x = gender @@ -1295,7 +1295,7 @@ salary:integer |x:keyword|gender:keyword |min_sl:integer |emp_no:integer ; overridingGroupings -required_capability: inlinestats_v9 +required_capability: inlinestats_v11 FROM employees | INLINESTATS min_sl = MIN(salary) BY x = gender, x = languages @@ -1314,7 +1314,7 @@ salary:integer |x:integer |gender:keyword |min_sl:integer |emp_no:integer ; overridingExpressionGroupings -required_capability: inlinestats_v9 +required_capability: inlinestats_v11 FROM employees | INLINESTATS min_sl = MIN(salary) BY x = TO_LOWER(gender), x = CONCAT(gender, gender) @@ -1333,7 +1333,7 @@ salary:integer |x:keyword |gender:keyword |min_sl:integer |emp_no:integer ; reusingEvalExpressions_UsedInGroupings -required_capability: inlinestats_v9 +required_capability: inlinestats_v11 FROM employees | KEEP salary, gender, emp_no @@ -1352,7 +1352,7 @@ salary:integer |gender:keyword |emp_no:integer |min_sl:integer | x:keyword ; statsBeforeInlinestatsWithTopAndBucket1 -required_capability: inlinestats_v9 +required_capability: inlinestats_v11 FROM books | STATS avg_rating = AVG(ratings) BY decade = BUCKET(year, 10) @@ -1372,7 +1372,7 @@ avg_rating:double | decade:double | decades:double ; statsBeforeInlinestatsWithTopAndBucket2 -required_capability: inlinestats_v9 +required_capability: inlinestats_v11 FROM sample_data | STATS total_duration = SUM(event_duration) BY day = BUCKET(@timestamp, 1 HOUR) @@ -1388,7 +1388,7 @@ total_duration:long | day:date | days:date evalBeforeInlinestatsAndKeepAfter1 -required_capability: inlinestats_v9 +required_capability: inlinestats_v11 FROM employees | WHERE still_hired == false @@ -1408,7 +1408,7 @@ emp_no:integer |still_hired:boolean|totalK:long|count:long ; evalBeforeInlinestatsAndKeepAfter2 -required_capability: inlinestats_v9 +required_capability: inlinestats_v11 FROM employees | EVAL salaryK = salary/1000 @@ -1428,7 +1428,7 @@ emp_no:integer |still_hired:boolean|total:long|count:long ; evalBeforeInlinestatsAndKeepAfter3 -required_capability: inlinestats_v9 +required_capability: inlinestats_v11 FROM employees | EVAL salaryK = salary/1000 @@ -1447,7 +1447,7 @@ emp_no:integer |still_hired:boolean|total:long ; evalBeforeInlinestatsAndKeepAfter4 -required_capability: inlinestats_v9 +required_capability: inlinestats_v11 FROM employees | EVAL salaryK = salary/1000 @@ -1466,7 +1466,7 @@ emp_no:integer |still_hired:boolean|count:long ; evalBeforeInlinestatsAndKeepAfter5 -required_capability: inlinestats_v9 +required_capability: inlinestats_v11 ROW salary = 12300, emp_no = 5, gender = "F" | EVAL salaryK = salary/1000 @@ -1477,3 +1477,1105 @@ ROW salary = 12300, emp_no = 5, gender = "F" emp_no:integer 5 ; + +//////////////////////////////// +// Union types tests +//////////////////////////////// + +singleIndexIpInlinestats +required_capability: inlinestats_v11 + +FROM sample_data +| EVAL client_ip = client_ip::ip +| INLINESTATS count=count(*) BY client_ip +| SORT count DESC, client_ip ASC +| KEEP count, client_ip +; + +count:long | client_ip:ip +4 |172.21.3.15 +4 |172.21.3.15 +4 |172.21.3.15 +4 |172.21.3.15 +1 |172.21.0.5 +1 |172.21.2.113 +1 |172.21.2.162 +; + +singleIndexIpStringInlinestats +required_capability: inlinestats_v11 + +FROM sample_data_str +| EVAL client_ip = client_ip::ip +| INLINESTATS count=count(*) BY client_ip +| SORT count DESC, client_ip ASC +| KEEP count, client_ip +; + +count:long | client_ip:ip +4 |172.21.3.15 +4 |172.21.3.15 +4 |172.21.3.15 +4 |172.21.3.15 +1 |172.21.0.5 +1 |172.21.2.113 +1 |172.21.2.162 +; + +singleIndexIpStringInlinestats Inline +required_capability: inlinestats_v11 + +FROM sample_data_str +| INLINESTATS count=count(*) BY client_ip::ip +| INLINESTATS mc=count(count) BY count +| SORT mc DESC, count ASC +| KEEP mc, count +; + +mc:l | count:l +4 |4 +4 |4 +4 |4 +4 |4 +3 |1 +3 |1 +3 |1 +; + +singleIndexIpStringInlinestatsInline_SecondStats +// same as above but the second agg is a STATS one +required_capability: inlinestats_v11 + +FROM sample_data_str +| INLINESTATS count=count(*) BY client_ip::ip +| STATS mc=count(count) BY count +| SORT mc DESC, count ASC +| KEEP mc, count +; + +mc:l | count:l +4 |4 +3 |1 +; + +multiIndexIpStringInlinestats +required_capability: inlinestats_v11 + +FROM sample_data, sample_data_str +| EVAL client_ip = client_ip::ip +| INLINESTATS count=count(*) BY client_ip +| SORT count DESC, client_ip ASC +| KEEP count, client_ip +; + +count:long | client_ip:ip +8 |172.21.3.15 +8 |172.21.3.15 +8 |172.21.3.15 +8 |172.21.3.15 +8 |172.21.3.15 +8 |172.21.3.15 +8 |172.21.3.15 +8 |172.21.3.15 +2 |172.21.0.5 +2 |172.21.0.5 +2 |172.21.2.113 +2 |172.21.2.113 +2 |172.21.2.162 +2 |172.21.2.162 +; + +multiIndexIpStringRenameInlinestats +required_capability: inlinestats_v11 + +FROM sample_data, sample_data_str +| EVAL host_ip = client_ip::ip +| INLINESTATS count=count(*) BY host_ip +| SORT count DESC, host_ip ASC +| KEEP count, host_ip +; + +count:long | host_ip:ip +8 |172.21.3.15 +8 |172.21.3.15 +8 |172.21.3.15 +8 |172.21.3.15 +8 |172.21.3.15 +8 |172.21.3.15 +8 |172.21.3.15 +8 |172.21.3.15 +2 |172.21.0.5 +2 |172.21.0.5 +2 |172.21.2.113 +2 |172.21.2.113 +2 |172.21.2.162 +2 |172.21.2.162 +; + +multiIndexIpStringRenameToStringInlinestats +required_capability: inlinestats_v11 + +FROM sample_data, sample_data_str +| EVAL host_ip = TO_STRING(TO_IP(client_ip)) +| INLINESTATS count=count(*) BY host_ip +| SORT count DESC, host_ip ASC +| KEEP count, host_ip +; + +count:long | host_ip:keyword +8 |172.21.3.15 +8 |172.21.3.15 +8 |172.21.3.15 +8 |172.21.3.15 +8 |172.21.3.15 +8 |172.21.3.15 +8 |172.21.3.15 +8 |172.21.3.15 +2 |172.21.0.5 +2 |172.21.0.5 +2 |172.21.2.113 +2 |172.21.2.113 +2 |172.21.2.162 +2 |172.21.2.162 +; + +multiIndexIpStringInlinestats Drop +required_capability: inlinestats_v11 + +FROM sample_data, sample_data_str +| INLINESTATS count=count(*) BY client_ip::ip +| KEEP count +| SORT count DESC +; + +count:long +8 +8 +8 +8 +8 +8 +8 +8 +2 +2 +2 +2 +2 +2 +; + +multiIndexIpStringInlinestats Inline +required_capability: inlinestats_v11 + +FROM sample_data, sample_data_str +| INLINESTATS count=count(*) BY client_ip = TO_IP(client_ip) +| SORT count DESC, client_ip ASC +| KEEP count, client_ip +; + +count:long | client_ip:ip +8 |172.21.3.15 +8 |172.21.3.15 +8 |172.21.3.15 +8 |172.21.3.15 +8 |172.21.3.15 +8 |172.21.3.15 +8 |172.21.3.15 +8 |172.21.3.15 +2 |172.21.0.5 +2 |172.21.0.5 +2 |172.21.2.113 +2 |172.21.2.113 +2 |172.21.2.162 +2 |172.21.2.162 +; + +statsUnionAggInline +required_capability: inlinestats_v11 + +FROM sample_data, sample_data_str +| INLINESTATS count = COUNT(CIDR_MATCH(TO_IP(client_ip), "172.21.0.0/24") OR NULL) + BY + @timestamp = DATE_TRUNC(10 minutes, @timestamp) +| SORT count DESC, @timestamp ASC, event_duration +| LIMIT 8 +; + +client_ip:unsupported |event_duration:long| message:keyword | count:long | @timestamp:date +null |1232382 |Disconnected |2 |2023-10-23T13:30:00.000Z +null |1232382 |Disconnected |2 |2023-10-23T13:30:00.000Z +null |3450233 |Connected to 10.1.0.3|0 |2023-10-23T12:10:00.000Z +null |3450233 |Connected to 10.1.0.3|0 |2023-10-23T12:10:00.000Z +null |2764889 |Connected to 10.1.0.2|0 |2023-10-23T12:20:00.000Z +null |2764889 |Connected to 10.1.0.2|0 |2023-10-23T12:20:00.000Z +null |725448 |Connection error |0 |2023-10-23T13:50:00.000Z +null |725448 |Connection error |0 |2023-10-23T13:50:00.000Z +; + +multiIndexIpInlinestats NonPushableCount +required_capability: inlinestats_v11 + +FROM sample_data, sample_data_str +| INLINESTATS count=count(client_ip::ip) +| SORT event_duration DESC +| LIMIT 2 +; + + @timestamp:date |client_ip:unsupported|event_duration:long|message:keyword |count:long +2023-10-23T13:52:55.015Z|null |8268153 |Connection error|14 +2023-10-23T13:52:55.015Z|null |8268153 |Connection error|14 +; + +multiIndexIpInlinestats NonPushableCountEval +required_capability: inlinestats_v11 + +FROM sample_data, sample_data_str +| EVAL client_ip = client_ip::ip +| INLINESTATS count=count(client_ip) +| SORT event_duration DESC +| LIMIT 2 +; + + @timestamp:date |event_duration:long|message:keyword |client_ip:ip |count:long +2023-10-23T13:52:55.015Z|8268153 |Connection error|172.21.3.15 |14 +2023-10-23T13:52:55.015Z|8268153 |Connection error|172.21.3.15 |14 +; + +multiIndexIpInlinestats NonPushableCountWithFilter +required_capability: inlinestats_v11 + +FROM sample_data, sample_data_ts_long +| INLINESTATS count_matching=count(@timestamp::long) WHERE @timestamp::long >= 1698069301543, + total_count=count(@timestamp::long) +| SORT client_ip +| LIMIT 4 +; + +@timestamp:unsupported| client_ip:ip|event_duration:long | message:keyword |count_matching:long | total_count:long +null |172.21.0.5 |1232382 |Disconnected |2 |14 +null |172.21.0.5 |1232382 |Disconnected |2 |14 +null |172.21.2.113 |2764889 |Connected to 10.1.0.2|2 |14 +null |172.21.2.113 |2764889 |Connected to 10.1.0.2|2 |14 +; + +multiIndexIpInlinestats PushableCount-Ignore +required_capability: inlinestats_v11 + +FROM sample_data, sample_data_ts_long +| INLINESTATS count=count(@timestamp::long) +| KEEP count +| LIMIT 2 +; + +count:long +14 +14 +; + +multiIndexIpInlinestats PushableCountEval-Ignore +required_capability: inlinestats_v11 + +FROM sample_data, sample_data_ts_long +| EVAL @timestamp = @timestamp::long +| INLINESTATS count=count(@timestamp) +| KEEP count +| LIMIT 2 +; + +count:long +14 +14 +; + +multiIndexIpStringInlinestats Inline2 +required_capability: inlinestats_v11 + +FROM sample_data, sample_data_str +| INLINESTATS count=count(*) BY client_ip::ip +| SORT count DESC, `client_ip::ip` ASC +| KEEP count, `client_ip::ip` +; + +count:long | client_ip::ip:ip +8 |172.21.3.15 +8 |172.21.3.15 +8 |172.21.3.15 +8 |172.21.3.15 +8 |172.21.3.15 +8 |172.21.3.15 +8 |172.21.3.15 +8 |172.21.3.15 +2 |172.21.0.5 +2 |172.21.0.5 +2 |172.21.2.113 +2 |172.21.2.113 +2 |172.21.2.162 +2 |172.21.2.162 +; + +multiIndexIpStringInlinestats Inline3 +required_capability: inlinestats_v11 + +FROM sample_data, sample_data_str +| INLINESTATS count=count(*) BY client_ip::ip +| INLINESTATS mc=count(count) BY count +| SORT mc DESC, count ASC, event_duration +| KEEP mc, count, event_duration +; + +mc:l | count:l | event_duration:l +8 |8 |725448 +8 |8 |725448 +8 |8 |1756467 +8 |8 |1756467 +8 |8 |5033755 +8 |8 |5033755 +8 |8 |8268153 +8 |8 |8268153 +6 |2 |1232382 +6 |2 |1232382 +6 |2 |2764889 +6 |2 |2764889 +6 |2 |3450233 +6 |2 |3450233 +; + +ultiIndexIpStringInlinestats Inline4 +required_capability: inlinestats_v11 + +FROM sample_data, sample_data_str +| INLINESTATS count=count(*) BY client_ip::ip +| LIMIT 5 +| INLINESTATS mc=count(count) BY count +| SORT mc DESC, count ASC, event_duration +| KEEP mc, count, event_duration +; + +mc:l | count:l | event_duration:l +4 |4 |725448 +4 |4 |1756467 +4 |4 |5033755 +4 |4 |8268153 +1 |1 |1232382 +; + +multiIndexWhereIpStringInlinestats +required_capability: inlinestats_v11 + +FROM sample_data, sample_data_str +| WHERE STARTS_WITH(TO_STRING(client_ip), "172.21.2") +| INLINESTATS count=count(*) BY message +| SORT count DESC, message ASC +| KEEP count, message +; + +count:long | message:keyword +2 |Connected to 10.1.0.2 +2 |Connected to 10.1.0.2 +2 |Connected to 10.1.0.3 +2 |Connected to 10.1.0.3 +; + +multiIndexTsLongInlinestats +required_capability: inlinestats_v11 + +FROM sample_data, sample_data_ts_long +| EVAL @timestamp = DATE_TRUNC(1 hour, TO_DATETIME(@timestamp)) +| INLINESTATS count=count(*) BY @timestamp +| SORT count DESC, @timestamp ASC +| KEEP count, @timestamp +; + +count:long | @timestamp:date +10 |2023-10-23T13:00:00.000Z +10 |2023-10-23T13:00:00.000Z +10 |2023-10-23T13:00:00.000Z +10 |2023-10-23T13:00:00.000Z +10 |2023-10-23T13:00:00.000Z +10 |2023-10-23T13:00:00.000Z +10 |2023-10-23T13:00:00.000Z +10 |2023-10-23T13:00:00.000Z +10 |2023-10-23T13:00:00.000Z +10 |2023-10-23T13:00:00.000Z +4 |2023-10-23T12:00:00.000Z +4 |2023-10-23T12:00:00.000Z +4 |2023-10-23T12:00:00.000Z +4 |2023-10-23T12:00:00.000Z +; + +multiIndexTsNanosToDatetimeInlinestats +required_capability: inlinestats_v11 + +FROM sample_data, sample_data_ts_nanos +| EVAL @timestamp = DATE_TRUNC(1 hour, TO_DATETIME(@timestamp)) +| INLINESTATS count=count(*) BY @timestamp +| SORT count DESC, @timestamp ASC +| KEEP count, @timestamp +| STATS count=count(*) BY @timestamp +; + +count:long | @timestamp:date +10 | 2023-10-23T13:00:00.000Z +4 | 2023-10-23T12:00:00.000Z +; + +Multi Index millis to nanos stats +required_capability: inlinestats_v11 + +FROM sample_data, sample_data_ts_nanos +| EVAL @timestamp = DATE_TRUNC(1 hour, TO_DATE_NANOS(@timestamp)) +| INLINESTATS count=count(*) BY @timestamp +| SORT count DESC, @timestamp ASC +| KEEP count, @timestamp +| STATS count=count(*) BY @timestamp +; + +count:long | @timestamp:date_nanos +10 | 2023-10-23T13:00:00.000Z +4 | 2023-10-23T12:00:00.000Z +; + + +multiIndexTsLongInlinestats Drop-Ignore +required_capability: inlinestats_v11 + +FROM sample_data, sample_data_ts_long, sample_data_ts_nanos +| INLINESTATS count=count(*) BY @timestamp::datetime +| KEEP count +| LIMIT 2 +; + +count:long +3 +3 +; + +multiIndexTsLongInlinestats Inline2 +required_capability: inlinestats_v11 + +FROM sample_data, sample_data_ts_long, sample_data_ts_nanos +| INLINESTATS count=count(*) BY @timestamp::datetime +| SORT `@timestamp::datetime` DESC, event_duration +| LIMIT 6 +; + +@timestamp:unsupported| client_ip:ip|event_duration:long| message:keyword | count:long| @timestamp::datetime:datetime +null |172.21.3.15 |1756467 |Connected to 10.1.0.1|3 |2023-10-23T13:55:01.543Z +null |172.21.3.15 |1756467 |Connected to 10.1.0.1|3 |2023-10-23T13:55:01.543Z +null |172.21.3.15 |1756467 |Connected to 10.1.0.1|3 |2023-10-23T13:55:01.543Z +null |172.21.3.15 |5033755 |Connection error |3 |2023-10-23T13:53:55.832Z +null |172.21.3.15 |5033755 |Connection error |3 |2023-10-23T13:53:55.832Z +null |172.21.3.15 |5033755 |Connection error |3 |2023-10-23T13:53:55.832Z +; + +multiIndexTsLongInlinestats Inline3 +required_capability: inlinestats_v11 + +FROM sample_data, sample_data_ts_long +| INLINESTATS count=count(*) BY @timestamp::datetime +| INLINESTATS mc=count(count) BY count +| SORT client_ip DESC +| LIMIT 1 +| KEEP mc, count +; + +mc:l | count:l +14 | 2 +; + +multiIndexTsLongInlinestats Inlinestats +required_capability: inlinestats_v11 + +FROM sample_data, sample_data_ts_long +| EVAL ts = TO_STRING(@timestamp) +| INLINESTATS count = COUNT(*) BY ts +| INLINESTATS mc = COUNT(count) BY count +| SORT ts +| LIMIT 1 +; + +@timestamp:unsupported| client_ip:ip|event_duration:long| message:keyword | ts:keyword| mc:long | count:long +null |172.21.2.162 |3450233 |Connected to 10.1.0.3|1698063303360 |14 |1 +; + +multiIndexTsLongRenameInlinestats +required_capability: inlinestats_v11 + +FROM sample_data, sample_data_ts_long +| EVAL hour = DATE_TRUNC(1 hour, TO_DATETIME(@timestamp)) +| INLINESTATS count=count(*) BY hour +| SORT count DESC, hour ASC +| KEEP count, hour +; + +count:long | hour:date +10 |2023-10-23T13:00:00.000Z +10 |2023-10-23T13:00:00.000Z +10 |2023-10-23T13:00:00.000Z +10 |2023-10-23T13:00:00.000Z +10 |2023-10-23T13:00:00.000Z +10 |2023-10-23T13:00:00.000Z +10 |2023-10-23T13:00:00.000Z +10 |2023-10-23T13:00:00.000Z +10 |2023-10-23T13:00:00.000Z +10 |2023-10-23T13:00:00.000Z +4 |2023-10-23T12:00:00.000Z +4 |2023-10-23T12:00:00.000Z +4 |2023-10-23T12:00:00.000Z +4 |2023-10-23T12:00:00.000Z +; + +multiIndexTsLongRenameToDatetimeToStringInlinestats +required_capability: inlinestats_v11 + +FROM sample_data, sample_data_ts_long +| EVAL hour = LEFT(TO_STRING(TO_DATETIME(@timestamp)), 13) +| INLINESTATS count=count(*) BY hour +| SORT count DESC, hour ASC +| STATS count_distinct(count) BY count, hour +| KEEP count, hour +; + +count:long | hour:keyword +10 | 2023-10-23T13 +4 | 2023-10-23T12 +; + +multiIndexTsLongRenameToStringInlinestats +required_capability: inlinestats_v11 + +FROM sample_data, sample_data_ts_long +| EVAL mess = LEFT(TO_STRING(@timestamp), 7) +| INLINESTATS count=count(*) BY mess +| SORT count DESC, mess DESC +| KEEP count, mess +; + +count:long | mess:keyword +7 |2023-10 +7 |2023-10 +7 |2023-10 +7 |2023-10 +7 |2023-10 +7 |2023-10 +7 |2023-10 +4 |1698069 +4 |1698069 +4 |1698069 +4 |1698069 +1 |1698068 +1 |1698064 +1 |1698063 +; + +multiIndexTsLongInlinestats Inline-Ignore +required_capability: inlinestats_v11 + +FROM sample_data, sample_data_ts_long +| INLINESTATS count=COUNT(*), max=MAX(TO_DATETIME(@timestamp)) +| KEEP count, max +| LIMIT 3 +; + +count:long | max:date +14 | 2023-10-23T13:55:01.543Z +14 | 2023-10-23T13:55:01.543Z +14 | 2023-10-23T13:55:01.543Z +; + +multiIndexTsLongInlinestats InlineDropped-Ignore +required_capability: inlinestats_v11 + +FROM sample_data, sample_data_ts_long +| INLINESTATS count=COUNT(*), max=MAX(TO_DATETIME(@timestamp)) +| KEEP count +| LIMIT 2 +; + +count:long +14 +14 +; + +multiIndexWhereTsLongInlinestats +required_capability: inlinestats_v11 + +FROM sample_data, sample_data_ts_long +| WHERE TO_LONG(@timestamp) < 1698068014937 +| INLINESTATS count=count(*) BY message +| SORT count DESC, message ASC +| KEEP count, message +; + +count:long | message:keyword +2 | Connected to 10.1.0.2 +2 | Connected to 10.1.0.2 +2 | Connected to 10.1.0.3 +2 | Connected to 10.1.0.3 +; + +multiIndexWhereIpStringTsLongInlinestats +required_capability: inlinestats_v11 + +FROM sample_data* +| WHERE TO_LONG(@timestamp) < 1698068014937 AND TO_STRING(client_ip) == "172.21.2.162" +| INLINESTATS count=count(*) BY message +| SORT count DESC, message ASC +| KEEP count, message +; + +count:long | message:keyword +3 | Connected to 10.1.0.3 +3 | Connected to 10.1.0.3 +3 | Connected to 10.1.0.3 +; + +multiIndexWhereIpStringLikeTsLongInlinestats +required_capability: inlinestats_v11 + +FROM sample_data* +| WHERE TO_LONG(@timestamp) < 1698068014937 AND TO_STRING(client_ip) LIKE "172.21.2.16?" +| INLINESTATS count=count(*) BY message +| SORT count DESC, message ASC +| KEEP count, message +; + +count:long | message:keyword +3 | Connected to 10.1.0.3 +3 | Connected to 10.1.0.3 +3 | Connected to 10.1.0.3 +; + +multiIndexInlinestats OfMultiTypedField +required_capability: inlinestats_v11 + +FROM apps, apps_short +| INLINESTATS s = sum(id::integer) +| SORT name, version +| LIMIT 5 +; + + id:unsupported| name:keyword |version:version| s:long +null |aaaaa |1 |210 +null |aaaaa |1 |210 +null |aaaaa |1.2.3.4 |210 +null |aaaaa |1.2.3.4 |210 +null |bbbbb |2.1 |210 +; + +inlineStatsUnionGroup +required_capability: inlinestats_v11 + +FROM sample_data, sample_data_ts_long +| INLINESTATS count = COUNT(*) + BY @timestamp = SUBSTRING(TO_STRING(@timestamp), 0, 7) +| SORT client_ip ASC, @timestamp ASC +| LIMIT 4 +; + +client_ip:ip | event_duration:long | message:keyword | count:long| @timestamp:keyword + 172.21.0.5 | 1232382 | Disconnected | 1 | 1698068 + 172.21.0.5 | 1232382 | Disconnected | 7 | 2023-10 +172.21.2.113 | 2764889 | Connected to 10.1.0.2 | 1 | 1698064 +172.21.2.113 | 2764889 | Connected to 10.1.0.2 | 7 | 2023-10 +; + +inlineStatsUnionGroupWithEval +required_capability: inlinestats_v11 + +FROM sample_data, sample_data_ts_long +| EVAL @timestamp = SUBSTRING(TO_STRING(@timestamp), 0, 7) +| INLINESTATS count = COUNT(*) BY @timestamp +| SORT client_ip ASC, @timestamp ASC +| LIMIT 4 +; + +client_ip:ip | event_duration:long | message:keyword | count:long| @timestamp:keyword + 172.21.0.5 | 1232382 | Disconnected | 1 | 1698068 + 172.21.0.5 | 1232382 | Disconnected | 7 | 2023-10 +172.21.2.113 | 2764889 | Connected to 10.1.0.2 | 1 | 1698064 +172.21.2.113 | 2764889 | Connected to 10.1.0.2 | 7 | 2023-10 +; + +inlineStatsUnionGroupTogether +required_capability: inlinestats_v11 + +FROM sample_data, sample_data_ts_long +| INLINESTATS count = COUNT(*) + BY @timestamp = TO_STRING(TO_DATETIME(@timestamp)) +| SORT client_ip ASC, @timestamp ASC +| LIMIT 4 +; + +client_ip:ip | event_duration:long | message:keyword |count:long| @timestamp:keyword + 172.21.0.5 | 1232382 | Disconnected |2 | 2023-10-23T13:33:34.937Z + 172.21.0.5 | 1232382 | Disconnected |2 | 2023-10-23T13:33:34.937Z +172.21.2.113 | 2764889 | Connected to 10.1.0.2 |2 | 2023-10-23T12:27:28.948Z +172.21.2.113 | 2764889 | Connected to 10.1.0.2 |2 | 2023-10-23T12:27:28.948Z +; + +multiIndexIndirectUseOfUnionTypesInInlinestats +required_capability: inlinestats_v11 + +FROM sample_data, sample_data_ts_long +| INLINESTATS foo = max(event_duration) BY client_ip +| RENAME event_duration AS e +| SORT client_ip ASC, e +| LIMIT 8 +; + +@timestamp:unsupported| e:long | message:keyword | foo:long| client_ip:ip +null |1232382 |Disconnected |1232382 |172.21.0.5 +null |1232382 |Disconnected |1232382 |172.21.0.5 +null |2764889 |Connected to 10.1.0.2|2764889 |172.21.2.113 +null |2764889 |Connected to 10.1.0.2|2764889 |172.21.2.113 +null |3450233 |Connected to 10.1.0.3|3450233 |172.21.2.162 +null |3450233 |Connected to 10.1.0.3|3450233 |172.21.2.162 +null |725448 |Connection error |8268153 |172.21.3.15 +null |725448 |Connection error |8268153 |172.21.3.15 +; + + +multiIndexIndirectUseOfUnionTypesInInlineStats +required_capability: inlinestats_v11 + +FROM sample_data, sample_data_ts_long +| INLINESTATS foo = max(event_duration) +| SORT client_ip ASC, event_duration +| LIMIT 1 +; + +@timestamp:unsupported | client_ip:ip | event_duration:long | message:keyword | foo:long + null | 172.21.0.5 | 1232382 | Disconnected | 8268153 +; + +shortIntegerWideningInlinestats +required_capability: inlinestats_v11 + +FROM apps, apps_short +| EVAL id = id::integer +| INLINESTATS count=count() BY name, id +| KEEP id, name, count +| SORT id ASC, name ASC +| LIMIT 5 +; + +id:integer | name:keyword | count:long +1 |aaaaa |2 +1 |aaaaa |2 +2 |bbbbb |2 +2 |bbbbb |2 +3 |ccccc |2 +; + +ImplicitCastingMultiTypedFieldsInlinestats ByNumeric +required_capability: inlinestats_v11 + +FROM employees, employees_incompatible +| INLINESTATS x=max(hire_date), y = min(hire_date) BY languages = languages::integer +| SORT languages +| KEEP emp_no, languages, x, y +| LIMIT 3 +; + +emp_no:unsupported|languages:integer| x:date_nanos | y:date_nanos +null |1 |1999-04-30T00:00:00.000Z|1985-02-18T00:00:00.000Z +null |1 |1999-04-30T00:00:00.000Z|1985-02-18T00:00:00.000Z +null |1 |1999-04-30T00:00:00.000Z|1985-02-18T00:00:00.000Z +; + +ImplicitCastingMultiTypedFieldsInlinestats ByNumericWithFilter +required_capability: inlinestats_v11 + +FROM employees, employees_incompatible +| INLINESTATS x=max(hire_date) where hire_date < "1993-08-03", y = min(hire_date) where hire_date >= "1993-08-03" BY languages = languages::integer +| SORT languages +| LIMIT 3 +| KEEP emp_no, languages, x, y +; + +emp_no:unsupported|languages:integer| x:date_nanos | y:date_nanos +null |1 |1990-10-20T00:00:00.000Z|1994-05-21T00:00:00.000Z +null |1 |1990-10-20T00:00:00.000Z|1994-05-21T00:00:00.000Z +null |1 |1990-10-20T00:00:00.000Z|1994-05-21T00:00:00.000Z +; + +ImplicitCastingMultiTypedFieldsInlinestats ByDateNanos +required_capability: inlinestats_v11 + +FROM employees, employees_incompatible +| INLINESTATS x=count(emp_no::long), y=avg(salary_change::double), z=max(height::double) BY hire_date +| EVAL y = round(y, 1), z = round(z, 1) +| KEEP x, y, z, hire_date +| SORT hire_date +| LIMIT 12 +; + + x:long | y:double | z:double | hire_date:date_nanos +2 |null |1.9 |1985-02-18T00:00:00.000Z +2 |null |1.9 |1985-02-18T00:00:00.000Z +2 |null |2.0 |1985-02-24T00:00:00.000Z +2 |null |2.0 |1985-02-24T00:00:00.000Z +2 |3.3 |2.0 |1985-05-13T00:00:00.000Z +2 |3.3 |2.0 |1985-05-13T00:00:00.000Z +2 |0.2 |1.8 |1985-07-09T00:00:00.000Z +2 |0.2 |1.8 |1985-07-09T00:00:00.000Z +2 |3.6 |1.5 |1985-09-17T00:00:00.000Z +2 |3.6 |1.5 |1985-09-17T00:00:00.000Z +2 |-5.9 |1.8 |1985-10-14T00:00:00.000Z +2 |-5.9 |1.8 |1985-10-14T00:00:00.000Z +; + +ImplicitCastingMultiTypedFieldsInlinestats ByDateNanosWithFilter +required_capability: inlinestats_v11 + +FROM employees, employees_incompatible +| INLINESTATS x=count(emp_no::long) where hire_date > "1985-05-01", y=avg(salary_change::double) where hire_date > "1985-05-01", z=max(height::double) where hire_date > "1985-05-01" BY hire_date +| EVAL y = round(y, 1), z = round(z, 1) +| KEEP x, y, z, hire_date +| SORT hire_date +| LIMIT 10 +; + + x:long | y:double | z:double | hire_date:date_nanos +0 |null |null |1985-02-18T00:00:00.000Z +0 |null |null |1985-02-18T00:00:00.000Z +0 |null |null |1985-02-24T00:00:00.000Z +0 |null |null |1985-02-24T00:00:00.000Z +2 |3.3 |2.0 |1985-05-13T00:00:00.000Z +2 |3.3 |2.0 |1985-05-13T00:00:00.000Z +2 |0.2 |1.8 |1985-07-09T00:00:00.000Z +2 |0.2 |1.8 |1985-07-09T00:00:00.000Z +2 |3.6 |1.5 |1985-09-17T00:00:00.000Z +2 |3.6 |1.5 |1985-09-17T00:00:00.000Z +; + +ImplicitCastingMultiTypedMVFieldsInlinestats MaxMin +required_capability: inlinestats_v11 + +FROM date_nanos, date_nanos_union_types +| INLINESTATS max = MAX(millis), min = MIN(nanos) +| SORT millis DESC +| LIMIT 5 +; + +warningRegex:evaluation of \[FROM .*date_nanos.*date_nanos_union_types.*\] failed, treating result as null. Only first 20 failures recorded. +warningRegex:java.lang.IllegalArgumentException: milliSeconds \[-1457696696640\] are before the epoch in 1970 and cannot be converted to nanoseconds + + millis:date_nanos | nanos:date_nanos | num:long | max:date_nanos | min:date_nanos +2023-10-23T13:55:01.543123456Z|2023-10-23T13:55:01.543Z |1698069301543123456|2023-10-23T13:55:01.543123456Z|2023-01-23T13:55:01.543Z +2023-10-23T13:55:01.543123456Z|2023-10-23T13:55:01.543Z |1698069301543123456|2023-10-23T13:55:01.543123456Z|2023-01-23T13:55:01.543Z +2023-10-23T13:55:01.543Z |2023-10-23T13:55:01.543123456Z|1698069301543123456|2023-10-23T13:55:01.543123456Z|2023-01-23T13:55:01.543Z +2023-10-23T13:53:55.832987654Z|2023-10-23T13:53:55.832Z |1698069235832987654|2023-10-23T13:55:01.543123456Z|2023-01-23T13:55:01.543Z +2023-10-23T13:53:55.832Z |2023-10-23T13:53:55.832987654Z|1698069235832987654|2023-10-23T13:55:01.543123456Z|2023-01-23T13:55:01.543Z +; + +ImplicitCastingMultiTypedMVFieldsInlinestats Values +required_capability: inlinestats_v11 + +FROM date_nanos, date_nanos_union_types +| INLINESTATS c = MV_COUNT(VALUES(nanos)) +| SORT millis DESC +| LIMIT 5 +; + +warningRegex:evaluation of \[FROM .*date_nanos.*date_nanos_union_types.*\] failed, treating result as null. Only first 20 failures recorded. +warningRegex:java.lang.IllegalArgumentException: milliSeconds \[-1457696696640\] are before the epoch in 1970 and cannot be converted to nanoseconds + + millis:date_nanos | nanos:date_nanos | num:long | c:integer +2023-10-23T13:55:01.543123456Z|2023-10-23T13:55:01.543Z |1698069301543123456|19 +2023-10-23T13:55:01.543123456Z|2023-10-23T13:55:01.543Z |1698069301543123456|19 +2023-10-23T13:55:01.543Z |2023-10-23T13:55:01.543123456Z|1698069301543123456|19 +2023-10-23T13:53:55.832987654Z|2023-10-23T13:53:55.832Z |1698069235832987654|19 +2023-10-23T13:53:55.832Z |2023-10-23T13:53:55.832987654Z|1698069235832987654|19 +; + +ImplicitCastingMultiTypedDateTruncInlinestats By-Ignore +required_capability: inlinestats_v11 + +FROM employees, employees_incompatible +| KEEP emp_no, hire_date +| INLINESTATS c = count(emp_no::long) BY yr = DATE_TRUNC(1 year, hire_date) +| SORT yr DESC +| LIMIT 5 +; + +c:long | yr:date_nanos +2 | 1999-01-01T00:00:00.000Z +2 | 1997-01-01T00:00:00.000Z +2 | 1996-01-01T00:00:00.000Z +10 | 1995-01-01T00:00:00.000Z +8 | 1994-01-01T00:00:00.000Z +; + +ImplicitCastingMultiTypedDateTruncInlinestats ByWithFilter-Ignore +required_capability: inlinestats_v11 + +FROM employees, employees_incompatible +| KEEP emp_no, hire_date +| INLINESTATS c = count(emp_no::long) where hire_date > "1996-01-01" BY yr = DATE_TRUNC(1 year, hire_date) +| SORT yr DESC +| LIMIT 5 +; + +c:long | yr:date_nanos +2 | 1999-01-01T00:00:00.000Z +2 | 1997-01-01T00:00:00.000Z +2 | 1996-01-01T00:00:00.000Z +0 | 1995-01-01T00:00:00.000Z +0 | 1994-01-01T00:00:00.000Z +; + +ImplicitCastingMultiTypedDateTruncInlinestats ByWithEval-Ignore +required_capability: inlinestats_v11 + +FROM employees, employees_incompatible +| KEEP emp_no, hire_date +| EVAL yr = DATE_TRUNC(1 year, hire_date) +| INLINESTATS c = count(emp_no::long) BY yr +| SORT yr DESC +| LIMIT 5 +; + +c:long | yr:date_nanos +2 | 1999-01-01T00:00:00.000Z +2 | 1997-01-01T00:00:00.000Z +2 | 1996-01-01T00:00:00.000Z +10 | 1995-01-01T00:00:00.000Z +8 | 1994-01-01T00:00:00.000Z +; + +ImplicitCastingMultiTypedDateTruncInlinestats ByWithEvalWithFilter-Ignore +required_capability: inlinestats_v11 + +FROM employees, employees_incompatible +| KEEP emp_no, hire_date +| EVAL yr = DATE_TRUNC(1 year, hire_date) +| INLINESTATS c = count(emp_no::long) where hire_date > "1991-01-01" BY yr +| SORT yr DESC +| LIMIT 5 +; + +c:long | yr:date_nanos +2 | 1999-01-01T00:00:00.000Z +2 | 1997-01-01T00:00:00.000Z +2 | 1996-01-01T00:00:00.000Z +10 | 1995-01-01T00:00:00.000Z +8 | 1994-01-01T00:00:00.000Z +; + +ImplicitCastingMultiTypedBucketDateNanosByYear +required_capability: inlinestats_v11 + +FROM employees, employees_incompatible +| KEEP emp_no, hire_date, gender +| INLINESTATS c = count(*) BY yr = BUCKET(hire_date, 1 year) +| SORT yr DESC, c, hire_date +| LIMIT 10 +; + +emp_no:unsupported| hire_date:date_nanos |gender:unsupported| c:long | yr:date_nanos +null |1999-04-30T00:00:00.000Z|null |2 |1999-01-01T00:00:00.000Z +null |1999-04-30T00:00:00.000Z|null |2 |1999-01-01T00:00:00.000Z +null |1997-05-19T00:00:00.000Z|null |2 |1997-01-01T00:00:00.000Z +null |1997-05-19T00:00:00.000Z|null |2 |1997-01-01T00:00:00.000Z +null |1996-11-05T00:00:00.000Z|null |2 |1996-01-01T00:00:00.000Z +null |1996-11-05T00:00:00.000Z|null |2 |1996-01-01T00:00:00.000Z +null |1995-01-27T00:00:00.000Z|null |10 |1995-01-01T00:00:00.000Z +null |1995-01-27T00:00:00.000Z|null |10 |1995-01-01T00:00:00.000Z +null |1995-03-13T00:00:00.000Z|null |10 |1995-01-01T00:00:00.000Z +null |1995-03-13T00:00:00.000Z|null |10 |1995-01-01T00:00:00.000Z +; + +ImplicitCastingMultiTypedBucketDateNanosByYearWithFilter +required_capability: inlinestats_v11 + +FROM employees, employees_incompatible +| KEEP emp_no, hire_date, gender +| INLINESTATS c = count(*) where hire_date > "1995-02-01" BY yr = BUCKET(hire_date, 1 year) +| SORT yr DESC, c, hire_date +| LIMIT 10 +; + +emp_no:unsupported| hire_date:date_nanos |gender:unsupported| c:long | yr:date_nanos +null |1999-04-30T00:00:00.000Z|null |2 |1999-01-01T00:00:00.000Z +null |1999-04-30T00:00:00.000Z|null |2 |1999-01-01T00:00:00.000Z +null |1997-05-19T00:00:00.000Z|null |2 |1997-01-01T00:00:00.000Z +null |1997-05-19T00:00:00.000Z|null |2 |1997-01-01T00:00:00.000Z +null |1996-11-05T00:00:00.000Z|null |2 |1996-01-01T00:00:00.000Z +null |1996-11-05T00:00:00.000Z|null |2 |1996-01-01T00:00:00.000Z +null |1995-01-27T00:00:00.000Z|null |8 |1995-01-01T00:00:00.000Z +null |1995-01-27T00:00:00.000Z|null |8 |1995-01-01T00:00:00.000Z +null |1995-03-13T00:00:00.000Z|null |8 |1995-01-01T00:00:00.000Z +null |1995-03-13T00:00:00.000Z|null |8 |1995-01-01T00:00:00.000Z +; + +ImplicitCastingMultiTypedBucketDateNanosByMonth +required_capability: inlinestats_v11 + +FROM employees, employees_incompatible +| INLINESTATS c = count(*) BY mo = BUCKET(hire_date, 20, "1986-01-01", "1999-12-31") +| KEEP emp_no, hire_date, c, mo +| SORT mo DESC, c, hire_date +| LIMIT 10 +; + +emp_no:unsupported| hire_date:date_nanos | c:long | mo:date_nanos +null |1999-04-30T00:00:00.000Z|2 |1999-01-01T00:00:00.000Z +null |1999-04-30T00:00:00.000Z|2 |1999-01-01T00:00:00.000Z +null |1997-05-19T00:00:00.000Z|2 |1997-01-01T00:00:00.000Z +null |1997-05-19T00:00:00.000Z|2 |1997-01-01T00:00:00.000Z +null |1996-11-05T00:00:00.000Z|2 |1996-01-01T00:00:00.000Z +null |1996-11-05T00:00:00.000Z|2 |1996-01-01T00:00:00.000Z +null |1995-01-27T00:00:00.000Z|10 |1995-01-01T00:00:00.000Z +null |1995-01-27T00:00:00.000Z|10 |1995-01-01T00:00:00.000Z +null |1995-03-13T00:00:00.000Z|10 |1995-01-01T00:00:00.000Z +null |1995-03-13T00:00:00.000Z|10 |1995-01-01T00:00:00.000Z +; + +ImplicitCastingMultiTypedBucketDateNanosInBothInlinestats AndBy +required_capability: inlinestats_v11 + +FROM employees, employees_incompatible +| INLINESTATS c = count(*), b = BUCKET(hire_date, 1 year) + 1 year BY yr = BUCKET(hire_date, 1 year) +| SORT yr DESC, c, hire_date +| KEEP emp_no, hire_date, c, b, yr +| LIMIT 10 +; + +emp_no:unsupported| hire_date:date_nanos | c:long | b:date_nanos | yr:date_nanos +null |1999-04-30T00:00:00.000Z|2 |2000-01-01T00:00:00.000Z|1999-01-01T00:00:00.000Z +null |1999-04-30T00:00:00.000Z|2 |2000-01-01T00:00:00.000Z|1999-01-01T00:00:00.000Z +null |1997-05-19T00:00:00.000Z|2 |1998-01-01T00:00:00.000Z|1997-01-01T00:00:00.000Z +null |1997-05-19T00:00:00.000Z|2 |1998-01-01T00:00:00.000Z|1997-01-01T00:00:00.000Z +null |1996-11-05T00:00:00.000Z|2 |1997-01-01T00:00:00.000Z|1996-01-01T00:00:00.000Z +null |1996-11-05T00:00:00.000Z|2 |1997-01-01T00:00:00.000Z|1996-01-01T00:00:00.000Z +null |1995-01-27T00:00:00.000Z|10 |1996-01-01T00:00:00.000Z|1995-01-01T00:00:00.000Z +null |1995-01-27T00:00:00.000Z|10 |1996-01-01T00:00:00.000Z|1995-01-01T00:00:00.000Z +null |1995-03-13T00:00:00.000Z|10 |1996-01-01T00:00:00.000Z|1995-01-01T00:00:00.000Z +null |1995-03-13T00:00:00.000Z|10 |1996-01-01T00:00:00.000Z|1995-01-01T00:00:00.000Z +; + +ImplicitCastingMultiTypedBucketDateNanosInBothInlinestats AndByWithAlias +required_capability: inlinestats_v11 + +FROM employees, employees_incompatible +| KEEP emp_no, hire_date +| INLINESTATS c = count(*), b = yr + 1 year BY yr = BUCKET(hire_date, 1 year) +| SORT hire_date DESC +| LIMIT 10 +; + +emp_no:unsupported| hire_date:date_nanos | c:long | b:date_nanos | yr:date_nanos +null |1999-04-30T00:00:00.000Z|2 |2000-01-01T00:00:00.000Z|1999-01-01T00:00:00.000Z +null |1999-04-30T00:00:00.000Z|2 |2000-01-01T00:00:00.000Z|1999-01-01T00:00:00.000Z +null |1997-05-19T00:00:00.000Z|2 |1998-01-01T00:00:00.000Z|1997-01-01T00:00:00.000Z +null |1997-05-19T00:00:00.000Z|2 |1998-01-01T00:00:00.000Z|1997-01-01T00:00:00.000Z +null |1996-11-05T00:00:00.000Z|2 |1997-01-01T00:00:00.000Z|1996-01-01T00:00:00.000Z +null |1996-11-05T00:00:00.000Z|2 |1997-01-01T00:00:00.000Z|1996-01-01T00:00:00.000Z +null |1995-12-15T00:00:00.000Z|10 |1996-01-01T00:00:00.000Z|1995-01-01T00:00:00.000Z +null |1995-12-15T00:00:00.000Z|10 |1996-01-01T00:00:00.000Z|1995-01-01T00:00:00.000Z +null |1995-08-22T00:00:00.000Z|10 |1996-01-01T00:00:00.000Z|1995-01-01T00:00:00.000Z +null |1995-08-22T00:00:00.000Z|10 |1996-01-01T00:00:00.000Z|1995-01-01T00:00:00.000Z +; diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/union_types.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/union_types.csv-spec index 7c89864989b08..9ed5603c04202 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/union_types.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/union_types.csv-spec @@ -1398,62 +1398,6 @@ event_duration:long | _index:keyword | ts:date | ts_str: ; -inlineStatsUnionGroup-Ignore -required_capability: union_types -required_capability: inlinestats - -FROM sample_data, sample_data_ts_long -| INLINESTATS count = COUNT(*) - BY @timestamp = SUBSTRING(TO_STRING(@timestamp), 0, 7) -| SORT client_ip ASC, @timestamp ASC -| LIMIT 4 -; - -client_ip:ip | event_duration:long | message:keyword | @timestamp:keyword | count:long - 172.21.0.5 | 1232382 | Disconnected | 1698068 | 1 - 172.21.0.5 | 1232382 | Disconnected | 2023-10 | 7 -172.21.2.113 | 2764889 | Connected to 10.1.0.2 | 1698064 | 1 -172.21.2.113 | 2764889 | Connected to 10.1.0.2 | 2023-10 | 7 -; - -inlineStatsUnionGroupWithEval-Ignore -required_capability: union_types -required_capability: inlinestats - -FROM sample_data, sample_data_ts_long -| EVAL @timestamp = SUBSTRING(TO_STRING(@timestamp), 0, 7) -| INLINESTATS count = COUNT(*) BY @timestamp -| SORT client_ip ASC, @timestamp ASC -| LIMIT 4 -; - -client_ip:ip | event_duration:long | message:keyword | @timestamp:keyword | count:long - 172.21.0.5 | 1232382 | Disconnected | 1698068 | 1 - 172.21.0.5 | 1232382 | Disconnected | 2023-10 | 7 -172.21.2.113 | 2764889 | Connected to 10.1.0.2 | 1698064 | 1 -172.21.2.113 | 2764889 | Connected to 10.1.0.2 | 2023-10 | 7 -; - -inlineStatsUnionGroupTogether-Ignore -required_capability: union_types -required_capability: inlinestats - -FROM sample_data, sample_data_ts_long -| INLINESTATS count = COUNT(*) - BY @timestamp = TO_STRING(TO_DATETIME(@timestamp)) -| SORT client_ip ASC, @timestamp ASC -| LIMIT 4 -; - -client_ip:ip | event_duration:long | message:keyword | @timestamp:keyword | count:long - 172.21.0.5 | 1232382 | Disconnected | 2023-10-23T13:33:34.937Z | 2 - 172.21.0.5 | 1232382 | Disconnected | 2023-10-23T13:33:34.937Z | 2 -172.21.2.113 | 2764889 | Connected to 10.1.0.2 | 2023-10-23T12:27:28.948Z | 2 -172.21.2.113 | 2764889 | Connected to 10.1.0.2 | 2023-10-23T12:27:28.948Z | 2 -; - -# Once INLINESTATS supports expressions in agg functions and groups, convert the group in the inlinestats - multiIndexIndirectUseOfUnionTypesInSort required_capability: union_types FROM sample_data, sample_data_ts_long @@ -1628,18 +1572,6 @@ foo:long | client_ip:ip 8268153 | 172.21.3.15 ; -multiIndexIndirectUseOfUnionTypesInInlineStats-Ignore -required_capability: union_types -required_capability: inlinestats -FROM sample_data, sample_data_ts_long -| INLINESTATS foo = max(event_duration) -| SORT client_ip ASC -| LIMIT 1 -; - -@timestamp:unsupported | client_ip:ip | event_duration:long | message:keyword | foo:long - null | 172.21.0.5 | 1232382 | Disconnected | 8268153 -; multiIndexIndirectUseOfUnionTypesInLookup-Ignore required_capability: union_types diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java index 6d88b479b59f4..b5ee37b70035f 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java @@ -967,7 +967,7 @@ public enum Cap { * Fixes a series of issues with inlinestats which had an incomplete implementation after lookup and inlinestats * were refactored. */ - INLINESTATS_V9(EsqlPlugin.INLINESTATS_FEATURE_FLAG), + INLINESTATS_V11(EsqlPlugin.INLINESTATS_FEATURE_FLAG), /** * Support partial_results diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java index 7307285ec37a7..bb46ce5fbce41 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java @@ -4222,7 +4222,7 @@ public void testGroupingOverridesInStats() { } public void testGroupingOverridesInInlinestats() { - assumeTrue("INLINESTATS required", EsqlCapabilities.Cap.INLINESTATS_V9.isEnabled()); + assumeTrue("INLINESTATS required", EsqlCapabilities.Cap.INLINESTATS_V11.isEnabled()); verifyUnsupported(""" from test | inlinestats MIN(salary) BY x = languages, x = x + 1 diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/PropagateInlineEvalsTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/PropagateInlineEvalsTests.java index 3b183c803ecd1..662ca51ab564d 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/PropagateInlineEvalsTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/PropagateInlineEvalsTests.java @@ -83,7 +83,7 @@ public static void init() { * \_StubRelation[[emp_no{f}#11, languages{f}#14, gender{f}#13, y{r}#10]] */ public void testGroupingAliasingMoved_To_LeftSideOfJoin() { - assumeTrue("Requires INLINESTATS", EsqlCapabilities.Cap.INLINESTATS_V9.isEnabled()); + assumeTrue("Requires INLINESTATS", EsqlCapabilities.Cap.INLINESTATS_V11.isEnabled()); var plan = plan(""" from test | keep emp_no, languages, gender @@ -126,7 +126,7 @@ public void testGroupingAliasingMoved_To_LeftSideOfJoin() { * {r}#21]] */ public void testGroupingAliasingMoved_To_LeftSideOfJoin_WithExpression() { - assumeTrue("Requires INLINESTATS", EsqlCapabilities.Cap.INLINESTATS_V9.isEnabled()); + assumeTrue("Requires INLINESTATS", EsqlCapabilities.Cap.INLINESTATS_V11.isEnabled()); var plan = plan(""" from test | keep emp_no, languages, gender, last_name, first_name diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/session/FieldNameUtilsTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/session/FieldNameUtilsTests.java index 58934fe68f8cb..10eb398e2f37d 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/session/FieldNameUtilsTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/session/FieldNameUtilsTests.java @@ -35,7 +35,7 @@ public void testBasicFromCommand() { } public void testBasicFromCommandWithInlinestats() { - assumeTrue("INLINESTATS required", EsqlCapabilities.Cap.INLINESTATS_V9.isEnabled()); + assumeTrue("INLINESTATS required", EsqlCapabilities.Cap.INLINESTATS_V11.isEnabled()); assertFieldNames("from test | inlinestats max(salary) by gender", ALL_FIELDS); } @@ -44,7 +44,7 @@ public void testBasicFromCommandWithMetadata() { } public void testBasicFromCommandWithMetadata_AndInlinestats() { - assumeTrue("INLINESTATS required", EsqlCapabilities.Cap.INLINESTATS_V9.isEnabled()); + assumeTrue("INLINESTATS required", EsqlCapabilities.Cap.INLINESTATS_V11.isEnabled()); assertFieldNames("from test metadata _index, _id, _version | inlinestats max(salary)", ALL_FIELDS); } @@ -321,7 +321,7 @@ public void testLimitZero() { } public void testLimitZero_WithInlinestats() { - assumeTrue("INLINESTATS required", EsqlCapabilities.Cap.INLINESTATS_V9.isEnabled()); + assumeTrue("INLINESTATS required", EsqlCapabilities.Cap.INLINESTATS_V11.isEnabled()); assertFieldNames(""" FROM employees | INLINESTATS COUNT(*), MAX(salary) BY gender @@ -336,7 +336,7 @@ public void testDocsDropHeight() { } public void testDocsDropHeight_WithInlinestats() { - assumeTrue("INLINESTATS required", EsqlCapabilities.Cap.INLINESTATS_V9.isEnabled()); + assumeTrue("INLINESTATS required", EsqlCapabilities.Cap.INLINESTATS_V11.isEnabled()); assertFieldNames(""" FROM employees | DROP height @@ -352,7 +352,7 @@ public void testDocsDropHeightWithWildcard() { } public void testDocsDropHeightWithWildcard_AndInlinestats() { - assumeTrue("INLINESTATS required", EsqlCapabilities.Cap.INLINESTATS_V9.isEnabled()); + assumeTrue("INLINESTATS required", EsqlCapabilities.Cap.INLINESTATS_V11.isEnabled()); assertFieldNames(""" FROM employees | INLINESTATS MAX(salary) BY gender @@ -519,7 +519,7 @@ public void testSortWithLimitOne_DropHeight() { } public void testSortWithLimitOne_DropHeight_WithInlinestats() { - assumeTrue("INLINESTATS required", EsqlCapabilities.Cap.INLINESTATS_V9.isEnabled()); + assumeTrue("INLINESTATS required", EsqlCapabilities.Cap.INLINESTATS_V11.isEnabled()); assertFieldNames("from employees | inlinestats avg(salary) by languages | sort languages | limit 1 | drop height*", ALL_FIELDS); } @@ -819,7 +819,7 @@ public void testFilterById() { } public void testFilterById_WithInlinestats() { - assumeTrue("INLINESTATS required", EsqlCapabilities.Cap.INLINESTATS_V9.isEnabled()); + assumeTrue("INLINESTATS required", EsqlCapabilities.Cap.INLINESTATS_V11.isEnabled()); assertFieldNames("FROM apps metadata _id | INLINESTATS max(rate) | WHERE _id == \"4\"", ALL_FIELDS); } @@ -1290,7 +1290,7 @@ public void testProjectDropPattern() { } public void testProjectDropPattern_WithInlinestats() { - assumeTrue("INLINESTATS required", EsqlCapabilities.Cap.INLINESTATS_V9.isEnabled()); + assumeTrue("INLINESTATS required", EsqlCapabilities.Cap.INLINESTATS_V11.isEnabled()); assertFieldNames(""" from test | inlinestats max(foo) by bar @@ -1373,7 +1373,7 @@ public void testCountAllAndOtherStatGrouped() { } public void testCountAllAndOtherStatGrouped_WithInlinestats() { - assumeTrue("INLINESTATS required", EsqlCapabilities.Cap.INLINESTATS_V9.isEnabled()); + assumeTrue("INLINESTATS required", EsqlCapabilities.Cap.INLINESTATS_V11.isEnabled()); assertFieldNames(""" from test | inlinestats c = count(*), min = min(emp_no) by languages @@ -1412,7 +1412,7 @@ public void testCountAllWithEval() { } public void testCountAllWithEval_AndInlinestats() { - assumeTrue("INLINESTATS required", EsqlCapabilities.Cap.INLINESTATS_V9.isEnabled()); + assumeTrue("INLINESTATS required", EsqlCapabilities.Cap.INLINESTATS_V11.isEnabled()); assertFieldNames(""" from test | rename languages as l @@ -1425,7 +1425,7 @@ public void testCountAllWithEval_AndInlinestats() { } public void testKeepAfterEval_AndInlinestats() { - assumeTrue("INLINESTATS required", EsqlCapabilities.Cap.INLINESTATS_V9.isEnabled()); + assumeTrue("INLINESTATS required", EsqlCapabilities.Cap.INLINESTATS_V11.isEnabled()); assertFieldNames(""" from test | rename languages as l @@ -1438,7 +1438,7 @@ public void testKeepAfterEval_AndInlinestats() { } public void testKeepBeforeEval_AndInlinestats() { - assumeTrue("INLINESTATS required", EsqlCapabilities.Cap.INLINESTATS_V9.isEnabled()); + assumeTrue("INLINESTATS required", EsqlCapabilities.Cap.INLINESTATS_V11.isEnabled()); assertFieldNames(""" from test | rename languages as l @@ -1451,7 +1451,7 @@ public void testKeepBeforeEval_AndInlinestats() { } public void testStatsBeforeEval_AndInlinestats() { - assumeTrue("INLINESTATS required", EsqlCapabilities.Cap.INLINESTATS_V9.isEnabled()); + assumeTrue("INLINESTATS required", EsqlCapabilities.Cap.INLINESTATS_V11.isEnabled()); assertFieldNames(""" from test | rename languages as l @@ -1463,7 +1463,7 @@ public void testStatsBeforeEval_AndInlinestats() { } public void testStatsBeforeInlinestats() { - assumeTrue("INLINESTATS required", EsqlCapabilities.Cap.INLINESTATS_V9.isEnabled()); + assumeTrue("INLINESTATS required", EsqlCapabilities.Cap.INLINESTATS_V11.isEnabled()); assertFieldNames(""" from test | stats min = min(salary) by languages @@ -1472,7 +1472,7 @@ public void testStatsBeforeInlinestats() { } public void testKeepBeforeInlinestats() { - assumeTrue("INLINESTATS required", EsqlCapabilities.Cap.INLINESTATS_V9.isEnabled()); + assumeTrue("INLINESTATS required", EsqlCapabilities.Cap.INLINESTATS_V11.isEnabled()); assertFieldNames(""" from test | keep languages, salary @@ -2846,7 +2846,7 @@ public void testForkAfterMvExpand() { } public void testForkBeforeInlineStatsIgnore() { - assumeTrue("INLINESTATS required", EsqlCapabilities.Cap.INLINESTATS_V9.isEnabled()); + assumeTrue("INLINESTATS required", EsqlCapabilities.Cap.INLINESTATS_V11.isEnabled()); assertTrue("FORK required", EsqlCapabilities.Cap.FORK_V9.isEnabled()); assertFieldNames(""" FROM employees @@ -2859,7 +2859,7 @@ public void testForkBeforeInlineStatsIgnore() { } public void testForkBranchWithInlineStatsIgnore() { - assumeTrue("INLINESTATS required", EsqlCapabilities.Cap.INLINESTATS_V9.isEnabled()); + assumeTrue("INLINESTATS required", EsqlCapabilities.Cap.INLINESTATS_V11.isEnabled()); assertTrue("FORK required", EsqlCapabilities.Cap.FORK_V9.isEnabled()); assertFieldNames(""" FROM employees @@ -2873,7 +2873,7 @@ public void testForkBranchWithInlineStatsIgnore() { } public void testForkAfterInlineStatsIgnore() { - assumeTrue("INLINESTATS required", EsqlCapabilities.Cap.INLINESTATS_V9.isEnabled()); + assumeTrue("INLINESTATS required", EsqlCapabilities.Cap.INLINESTATS_V11.isEnabled()); assertTrue("FORK required", EsqlCapabilities.Cap.FORK_V9.isEnabled()); assertFieldNames(""" FROM employees From 12b7b19233f34afcca3d9604b1c91a6e03786784 Mon Sep 17 00:00:00 2001 From: Andrei Stefan Date: Wed, 20 Aug 2025 15:21:24 +0300 Subject: [PATCH 02/11] Enable CCQ --- .../elasticsearch/xpack/core/ml/MachineLearningField.java | 2 +- .../elasticsearch/xpack/esql/ccq/MultiClusterSpecIT.java | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/MachineLearningField.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/MachineLearningField.java index a40babb2760fb..e2ae19555c6d1 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/MachineLearningField.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/MachineLearningField.java @@ -24,7 +24,7 @@ public final class MachineLearningField { public static final Setting AUTODETECT_PROCESS = Setting.boolSetting( "xpack.ml.autodetect_process", - true, + false, Setting.Property.NodeScope ); public static final Setting MAX_MODEL_MEMORY_LIMIT = Setting.memorySizeSetting( diff --git a/x-pack/plugin/esql/qa/server/multi-clusters/src/javaRestTest/java/org/elasticsearch/xpack/esql/ccq/MultiClusterSpecIT.java b/x-pack/plugin/esql/qa/server/multi-clusters/src/javaRestTest/java/org/elasticsearch/xpack/esql/ccq/MultiClusterSpecIT.java index 4b90aa1a69c80..d335f7bd60863 100644 --- a/x-pack/plugin/esql/qa/server/multi-clusters/src/javaRestTest/java/org/elasticsearch/xpack/esql/ccq/MultiClusterSpecIT.java +++ b/x-pack/plugin/esql/qa/server/multi-clusters/src/javaRestTest/java/org/elasticsearch/xpack/esql/ccq/MultiClusterSpecIT.java @@ -135,9 +135,9 @@ protected void shouldSkipTest(String testName) throws IOException { assumeFalse("can't test with _index metadata", (remoteMetadata == false) && hasIndexMetadata(testCase.query)); Version oldVersion = Version.min(Clusters.localClusterVersion(), Clusters.remoteClusterVersion()); assumeTrue("Test " + testName + " is skipped on " + oldVersion, isEnabled(testName, instructions, oldVersion)); - assumeFalse("INLINESTATS not yet supported in CCS", testCase.requiredCapabilities.contains(INLINESTATS.capabilityName())); - assumeFalse("INLINESTATS not yet supported in CCS", testCase.requiredCapabilities.contains(JOIN_PLANNING_V1.capabilityName())); - assumeFalse("INLINESTATS not yet supported in CCS", testCase.requiredCapabilities.contains(INLINESTATS_V11.capabilityName())); + //assumeFalse("INLINESTATS not yet supported in CCS", testCase.requiredCapabilities.contains(INLINESTATS.capabilityName())); + //assumeFalse("INLINESTATS not yet supported in CCS", testCase.requiredCapabilities.contains(JOIN_PLANNING_V1.capabilityName())); + //assumeFalse("INLINESTATS not yet supported in CCS", testCase.requiredCapabilities.contains(INLINESTATS_V11.capabilityName())); if (testCase.requiredCapabilities.contains(JOIN_LOOKUP_V12.capabilityName())) { assumeTrue( "LOOKUP JOIN not yet supported in CCS", From b2f9d01085a12c9a711c8e4fd43b6b8271699f20 Mon Sep 17 00:00:00 2001 From: Andrei Stefan Date: Tue, 26 Aug 2025 14:46:20 +0300 Subject: [PATCH 03/11] Update --- .../src/main/resources/inlinestats.csv-spec | 98 +++++++++---------- 1 file changed, 49 insertions(+), 49 deletions(-) diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/inlinestats.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/inlinestats.csv-spec index 85e613fe8bc5c..e282591693467 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/inlinestats.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/inlinestats.csv-spec @@ -1473,7 +1473,7 @@ emp_no:integer /////////////////////////// doubleFilterOnInlineStats -required_capability: inlinestats_v10 +required_capability: inlinestats_v11 from employees | keep salary, gender @@ -1499,7 +1499,7 @@ salary:integer | max1:integer | max2:integer | max3:integer | gender:keyword ; inlinestatsWithFiltering -required_capability: inlinestats_v10 +required_capability: inlinestats_v11 from employees | inlinestats max = max(salary), max_f = max(salary) where salary < 50000, max_a = max(salary) where salary > 100, min = min(salary), min_f = min(salary) where salary > 50000, min_a = min(salary) where salary > 100 @@ -1515,7 +1515,7 @@ max:integer |max_f:integer |max_a:integer | min:integer | min_f:integer | min_a: ; inlinestatsWithEverythingFiltered -required_capability: inlinestats_v10 +required_capability: inlinestats_v11 from employees | inlinestats max = max(salary), max_a = max(salary) where salary < 100, min = min(salary), min_a = min(salary) where salary > 99999 @@ -1533,7 +1533,7 @@ max:integer |max_a:integer|min:integer | min_a:integer | emp_no:integer|salary:i ; inlinestatsWithNullFilter -required_capability: inlinestats_v10 +required_capability: inlinestats_v11 from employees | inlinestats max = max(salary), max_a = max(salary) where null, min = min(salary), min_a = min(salary) where to_string(null) == "abc" @@ -1549,7 +1549,7 @@ max:integer |max_a:integer|min:integer | min_a:integer | emp_no:integer ; inlinestatsWithAllFiltersFalse -required_capability: inlinestats_v10 +required_capability: inlinestats_v11 from employees | inlinestats max = max(height.float) where false, min = min(height.float) where to_string(null) == "abc", @@ -1567,7 +1567,7 @@ emp_no:integer|salary:integer| max:double |min:double |count:long |count_distinc ; inlinestatsWithAllFiltersFalse_GroupByOneField -required_capability: inlinestats_v10 +required_capability: inlinestats_v11 from employees | inlinestats max = max(height.float) where false, min = min(height.float) where to_string(null) == "abc", @@ -1587,7 +1587,7 @@ from employees ; inlinestatsWithAllFiltersFalse_GroupByTwoFields -required_capability: inlinestats_v10 +required_capability: inlinestats_v11 from employees | inlinestats max = max(height.float) where false, min = min(height.float) where to_string(null) == "abc", @@ -1607,7 +1607,7 @@ from employees ; prunedInlinestatsFollowedByInlinestats_GroupByOneFieldEach_DifferentFields -required_capability: inlinestats_v10 +required_capability: inlinestats_v11 from employees | eval my_length = length(concat(first_name, null)) | inlinestats count = count(my_length) where false, @@ -1625,7 +1625,7 @@ from employees ; prunedInlinestatsFollowedByInlinestats_GroupByOneFieldEach_SameFields -required_capability: inlinestats_v10 +required_capability: inlinestats_v11 from employees | eval my_length = length(concat(first_name, null)) | inlinestats count = count(my_length) where false, @@ -1644,7 +1644,7 @@ from employees prunedInlinestatsFollowedByInlinestats_GroupByOneFieldOnSecondInlinestats-Ignore // values doesn't end up as null -required_capability: inlinestats_v10 +required_capability: inlinestats_v11 from employees | eval my_length = length(concat(first_name, null)) | inlinestats count = count(my_length) where false, @@ -1662,7 +1662,7 @@ from employees ; partial_PrunedInlinestatsFollowedByInlinestats_GroupByOneFieldOnFirstInlinestats -required_capability: inlinestats_v10 +required_capability: inlinestats_v11 from employees | eval my_length = length(concat(first_name, null)) | inlinestats count = count(is_rehired) where true, @@ -1688,7 +1688,7 @@ from employees partial_PrunedInlinestatsFollowedByInlinestats_GroupByOneFieldOnFirstInlinestats2 -required_capability: inlinestats_v10 +required_capability: inlinestats_v11 from employees | eval my_length = length(concat(first_name, null)) | inlinestats count = count(is_rehired) where true, @@ -1715,7 +1715,7 @@ from employees inlinestatsWithExpressionsAllFiltersFalse -required_capability: inlinestats_v10 +required_capability: inlinestats_v11 from employees | keep height.f*, emp_no | sort emp_no desc @@ -1733,7 +1733,7 @@ from employees ; inlinestatsWithFalseFilterAndGroup -required_capability: inlinestats_v10 +required_capability: inlinestats_v11 from employees | inlinestats max = max(height.float + 1) where null, count = count(height.float) + 2 where false @@ -1751,7 +1751,7 @@ from employees ; inlinestatsWithFalseFiltersAndGroups -required_capability: inlinestats_v10 +required_capability: inlinestats_v11 from employees | eval my_length = length(concat(first_name, null)) | inlinestats count_distinct = count_distinct(height.float + 1) where null, @@ -1777,7 +1777,7 @@ from employees ; inlinestatsWithFalseFiltersAndGroups_DropEvaledValue -required_capability: inlinestats_v10 +required_capability: inlinestats_v11 from employees | eval my_length = length(concat(first_name, null)) | inlinestats count_distinct = count_distinct(height.float + 1) where null, @@ -1804,7 +1804,7 @@ from employees ; inlinestatsWithMixedFiltersAndGroup -required_capability: inlinestats_v10 +required_capability: inlinestats_v11 from employees | eval my_length = length(concat(first_name, null)) | inlinestats count = count(my_length) where false, @@ -1824,7 +1824,7 @@ null |[0, 0, 0, 0] |[Berni, Chirstian, Amabile, Berni, Bojan, Chir prunedInlinestatsFollowedByinlinestats-Ignore // values doesn't end up as null -required_capability: inlinestats_v10 +required_capability: inlinestats_v11 from employees | eval my_length = length(concat(first_name, null)) | inlinestats count = count(my_length) where false, @@ -1847,7 +1847,7 @@ inlinestatsWithFalseFiltersFromRow // null |2 // null |3 // null |4 -required_capability: inlinestats_v10 +required_capability: inlinestats_v11 row x = null, a = 1, b = [2,3,4] | inlinestats c=max(a) where x by b @@ -1858,7 +1858,7 @@ null |1 |null |[2, 3, 4] ; inlinestatsWithBasicExpressionFiltered -required_capability: inlinestats_v10 +required_capability: inlinestats_v11 from employees | inlinestats max = max(salary), max_f = max(salary) where salary < 50000, min = min(salary), min_f = min(salary) where salary > 50000, @@ -1874,7 +1874,7 @@ from employees ; inlinestatsWithExpressionOverFilters -required_capability: inlinestats_v10 +required_capability: inlinestats_v11 from employees | inlinestats max = max(salary), max_f = max(salary) where salary < 50000, min = min(salary), min_f = min(salary) where salary > 50000, @@ -1893,7 +1893,7 @@ from employees inlinestatsWithExpressionOfExpressionsOverFilters -required_capability: inlinestats_v10 +required_capability: inlinestats_v11 from employees | inlinestats max = max(salary + 1), max_f = max(salary + 2) where salary < 50000, min = min(salary - 1), min_f = min(salary - 2) where salary > 50000, @@ -1911,7 +1911,7 @@ from employees ; inlinestatsWithSubstitutedExpressionOverFilters -required_capability: inlinestats_v10 +required_capability: inlinestats_v11 from employees | inlinestats sum = sum(salary), s_l = sum(salary) where salary < 50000, s_u = sum(salary) where salary > 50000, count = count(salary), c_l = count(salary) where salary < 50000, c_u = count(salary) where salary > 50000, @@ -1929,7 +1929,7 @@ from employees inlinestatsWithFilterAndGroupBy -required_capability: inlinestats_v10 +required_capability: inlinestats_v11 from employees | keep height, gender, is_rehired, emp_no | inlinestats m = max(height), m_f = max(height + 1) where gender == "M" OR is_rehired is null BY gender, is_rehired @@ -1953,7 +1953,7 @@ from employees inlinestatsWithFilterOnGroupBy -required_capability: inlinestats_v10 +required_capability: inlinestats_v11 from employees | inlinestats m_f = max(height) where gender == "M" BY gender | sort emp_no @@ -1975,7 +1975,7 @@ null |null ; inlinestatsWithGroupByLiteral -required_capability: inlinestats_v10 +required_capability: inlinestats_v11 from employees | inlinestats m = max(languages) by salary = 2 | sort salary @@ -1991,7 +1991,7 @@ from employees inlinestatsWithFilterOnSameColumn -required_capability: inlinestats_v10 +required_capability: inlinestats_v11 from employees | inlinestats m = max(languages), m_f = max(languages) where salary > 50000 by salary = 2 | sort salary @@ -2006,7 +2006,7 @@ from employees ; inlinestatsWithFilteringAndGrouping -required_capability: inlinestats_v10 +required_capability: inlinestats_v11 from employees | inlinestats c = count(), c_f = count(languages) where l > 1, m_f = max(height) where salary > 50000 @@ -2031,7 +2031,7 @@ from employees multiinlinestatsWithFiltering -required_capability: inlinestats_v10 +required_capability: inlinestats_v11 from employees | inlinestats c = count(), c_f = count(languages) where l > 1, m_f = max(height) where salary > 50000 @@ -2058,7 +2058,7 @@ from employees simpleCountOnFieldWithFilteringAndNoGrouping -required_capability: inlinestats_v10 +required_capability: inlinestats_v11 from employees | inlinestats c1 = count(emp_no) where emp_no < 10042 | keep emp_no, c1 @@ -2073,7 +2073,7 @@ emp_no:integer | c1:long ; simpleCountOnFieldWithFilteringOnDifferentFieldAndNoGrouping -required_capability: inlinestats_v10 +required_capability: inlinestats_v11 from employees | inlinestats c1 = count(hire_date) where emp_no < 10042 | keep emp_no, c1 @@ -2088,7 +2088,7 @@ emp_no:integer | c1:long ; simpleCountOnStarWithFilteringAndNoGrouping -required_capability: inlinestats_v10 +required_capability: inlinestats_v11 from employees | inlinestats c1 = count(*) where emp_no < 10042 | keep emp_no, c1 @@ -2103,7 +2103,7 @@ emp_no:integer | c1:long ; simpleCountWithFilteringAndNoGroupingOnFieldWithNulls -required_capability: inlinestats_v10 +required_capability: inlinestats_v11 from employees | inlinestats c1 = count(birth_date) where emp_no <= 10050 | keep emp_no, c1 @@ -2119,7 +2119,7 @@ emp_no:integer | c1:long simpleCountWithFilteringAndNoGroupingOnFieldWithMultivalues -required_capability: inlinestats_v10 +required_capability: inlinestats_v11 from employees | inlinestats c1 = count(job_positions) where emp_no <= 10003 | keep emp_no, c1 @@ -2134,7 +2134,7 @@ from employees ; commonFilterExtractionWithAliasing -required_capability: inlinestats_v10 +required_capability: inlinestats_v11 from employees | eval eno = emp_no | drop emp_no @@ -2152,7 +2152,7 @@ from employees ; commonFilterExtractionWithAliasAndOriginal -required_capability: inlinestats_v10 +required_capability: inlinestats_v11 from employees | eval eno = emp_no | inlinestats min_sal = min(salary) where eno <= 10010, @@ -2170,7 +2170,7 @@ from employees commonFilterExtractionWithAliasAndOriginalNeedingNormalization -required_capability: inlinestats_v10 +required_capability: inlinestats_v11 from employees | eval eno = emp_no | inlinestats min_sal = min(salary) where eno <= 10010, @@ -2190,7 +2190,7 @@ from employees ; commonFilterExtractionWithAliasAndOriginalNeedingNormalizationAndSimplification -required_capability: inlinestats_v10 +required_capability: inlinestats_v11 from employees | eval eno = emp_no | inlinestats min_sal = min(salary) where eno <= 10010, @@ -2210,7 +2210,7 @@ from employees ; filterIsAlwaysTrue -required_capability: inlinestats_v10 +required_capability: inlinestats_v11 FROM employees | inlinestats max = max(salary) WHERE salary > 0 | keep max, salary, emp_no @@ -2225,7 +2225,7 @@ FROM employees ; filterIsAlwaysFalse -required_capability: inlinestats_v10 +required_capability: inlinestats_v11 FROM employees | inlinestats max = max(salary) WHERE first_name == "" | sort emp_no @@ -2248,7 +2248,7 @@ null |Duangkaew filterSometimesMatches -required_capability: inlinestats_v10 +required_capability: inlinestats_v11 FROM employees | inlinestats max = max(salary) WHERE first_name IS NULL | sort emp_no @@ -2270,7 +2270,7 @@ FROM employees ; groupingFilterIsAlwaysTrue -required_capability: inlinestats_v10 +required_capability: inlinestats_v11 FROM employees | MV_EXPAND job_positions | inlinestats max = max(salary) WHERE salary > 0 BY job_positions = SUBSTRING(job_positions, 1, 1) @@ -2288,7 +2288,7 @@ FROM employees groupingFilterIsAlwaysFalse -required_capability: inlinestats_v10 +required_capability: inlinestats_v11 FROM employees | MV_EXPAND job_positions | inlinestats max = max(salary) WHERE first_name == "" BY job_positions = SUBSTRING(job_positions, 1, 1) @@ -2306,7 +2306,7 @@ null |48942 |Patricio |A |10012 groupingFilterSometimesMatches -required_capability: inlinestats_v10 +required_capability: inlinestats_v11 FROM employees | MV_EXPAND job_positions | inlinestats max = max(salary) WHERE first_name IS NULL BY job_positions = SUBSTRING(job_positions, 1, 1) @@ -2323,7 +2323,7 @@ FROM employees ; groupingByOrdinalsFilterIsAlwaysTrue -required_capability: inlinestats_v10 +required_capability: inlinestats_v11 FROM employees | inlinestats max = max(salary) WHERE salary > 0 BY job_positions | SORT job_positions ASC, salary DESC @@ -2339,7 +2339,7 @@ FROM employees ; groupingByOrdinalsFilterIsAlwaysFalse -required_capability: inlinestats_v10 +required_capability: inlinestats_v11 FROM employees | inlinestats max = max(salary) WHERE first_name == "" BY job_positions @@ -2356,7 +2356,7 @@ Mary |31120 |10011 |[Architect, Reporting Analyst, S ; groupingByOrdinalsFilterSometimesMatches -required_capability: inlinestats_v10 +required_capability: inlinestats_v11 FROM employees | keep salary, first_name, job_positions, emp_no @@ -2373,7 +2373,7 @@ FROM employees ; stdDevFilter -required_capability: inlinestats_v10 +required_capability: inlinestats_v11 FROM employees | inlinestats greater_than = STD_DEV(salary_change) WHERE languages > 3 , less_than = STD_DEV(salary_change) WHERE languages <= 3 @@ -2398,7 +2398,7 @@ FROM employees ; twoConsecutiveInlinestatsWithFalseFilters -required_capability: inlinestats_v10 +required_capability: inlinestats_v11 from employees | keep emp_no | sort emp_no From a1685d64210ba3b71780c4364996c4db68933bd1 Mon Sep 17 00:00:00 2001 From: elasticsearchmachine Date: Tue, 26 Aug 2025 11:54:42 +0000 Subject: [PATCH 04/11] [CI] Auto commit changes from spotless --- .../elasticsearch/xpack/esql/ccq/MultiClusterSpecIT.java | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/x-pack/plugin/esql/qa/server/multi-clusters/src/javaRestTest/java/org/elasticsearch/xpack/esql/ccq/MultiClusterSpecIT.java b/x-pack/plugin/esql/qa/server/multi-clusters/src/javaRestTest/java/org/elasticsearch/xpack/esql/ccq/MultiClusterSpecIT.java index d335f7bd60863..b08cb74eceb51 100644 --- a/x-pack/plugin/esql/qa/server/multi-clusters/src/javaRestTest/java/org/elasticsearch/xpack/esql/ccq/MultiClusterSpecIT.java +++ b/x-pack/plugin/esql/qa/server/multi-clusters/src/javaRestTest/java/org/elasticsearch/xpack/esql/ccq/MultiClusterSpecIT.java @@ -50,10 +50,7 @@ import static org.elasticsearch.xpack.esql.EsqlTestUtils.classpathResources; import static org.elasticsearch.xpack.esql.action.EsqlCapabilities.Cap.ENABLE_LOOKUP_JOIN_ON_REMOTE; import static org.elasticsearch.xpack.esql.action.EsqlCapabilities.Cap.FORK_V9; -import static org.elasticsearch.xpack.esql.action.EsqlCapabilities.Cap.INLINESTATS; -import static org.elasticsearch.xpack.esql.action.EsqlCapabilities.Cap.INLINESTATS_V11; import static org.elasticsearch.xpack.esql.action.EsqlCapabilities.Cap.JOIN_LOOKUP_V12; -import static org.elasticsearch.xpack.esql.action.EsqlCapabilities.Cap.JOIN_PLANNING_V1; import static org.elasticsearch.xpack.esql.action.EsqlCapabilities.Cap.METADATA_FIELDS_REMOTE_TEST; import static org.elasticsearch.xpack.esql.action.EsqlCapabilities.Cap.UNMAPPED_FIELDS; import static org.elasticsearch.xpack.esql.qa.rest.RestEsqlTestCase.hasCapabilities; @@ -135,9 +132,9 @@ protected void shouldSkipTest(String testName) throws IOException { assumeFalse("can't test with _index metadata", (remoteMetadata == false) && hasIndexMetadata(testCase.query)); Version oldVersion = Version.min(Clusters.localClusterVersion(), Clusters.remoteClusterVersion()); assumeTrue("Test " + testName + " is skipped on " + oldVersion, isEnabled(testName, instructions, oldVersion)); - //assumeFalse("INLINESTATS not yet supported in CCS", testCase.requiredCapabilities.contains(INLINESTATS.capabilityName())); - //assumeFalse("INLINESTATS not yet supported in CCS", testCase.requiredCapabilities.contains(JOIN_PLANNING_V1.capabilityName())); - //assumeFalse("INLINESTATS not yet supported in CCS", testCase.requiredCapabilities.contains(INLINESTATS_V11.capabilityName())); + // assumeFalse("INLINESTATS not yet supported in CCS", testCase.requiredCapabilities.contains(INLINESTATS.capabilityName())); + // assumeFalse("INLINESTATS not yet supported in CCS", testCase.requiredCapabilities.contains(JOIN_PLANNING_V1.capabilityName())); + // assumeFalse("INLINESTATS not yet supported in CCS", testCase.requiredCapabilities.contains(INLINESTATS_V11.capabilityName())); if (testCase.requiredCapabilities.contains(JOIN_LOOKUP_V12.capabilityName())) { assumeTrue( "LOOKUP JOIN not yet supported in CCS", From 85b41881cda20e9650c9f51a84a1bd49e19ab1cc Mon Sep 17 00:00:00 2001 From: Andrei Stefan Date: Tue, 26 Aug 2025 15:18:39 +0300 Subject: [PATCH 05/11] Disable CCQ --- .../elasticsearch/xpack/core/ml/MachineLearningField.java | 2 +- .../elasticsearch/xpack/esql/ccq/MultiClusterSpecIT.java | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/MachineLearningField.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/MachineLearningField.java index e2ae19555c6d1..a40babb2760fb 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/MachineLearningField.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/MachineLearningField.java @@ -24,7 +24,7 @@ public final class MachineLearningField { public static final Setting AUTODETECT_PROCESS = Setting.boolSetting( "xpack.ml.autodetect_process", - false, + true, Setting.Property.NodeScope ); public static final Setting MAX_MODEL_MEMORY_LIMIT = Setting.memorySizeSetting( diff --git a/x-pack/plugin/esql/qa/server/multi-clusters/src/javaRestTest/java/org/elasticsearch/xpack/esql/ccq/MultiClusterSpecIT.java b/x-pack/plugin/esql/qa/server/multi-clusters/src/javaRestTest/java/org/elasticsearch/xpack/esql/ccq/MultiClusterSpecIT.java index d335f7bd60863..4b90aa1a69c80 100644 --- a/x-pack/plugin/esql/qa/server/multi-clusters/src/javaRestTest/java/org/elasticsearch/xpack/esql/ccq/MultiClusterSpecIT.java +++ b/x-pack/plugin/esql/qa/server/multi-clusters/src/javaRestTest/java/org/elasticsearch/xpack/esql/ccq/MultiClusterSpecIT.java @@ -135,9 +135,9 @@ protected void shouldSkipTest(String testName) throws IOException { assumeFalse("can't test with _index metadata", (remoteMetadata == false) && hasIndexMetadata(testCase.query)); Version oldVersion = Version.min(Clusters.localClusterVersion(), Clusters.remoteClusterVersion()); assumeTrue("Test " + testName + " is skipped on " + oldVersion, isEnabled(testName, instructions, oldVersion)); - //assumeFalse("INLINESTATS not yet supported in CCS", testCase.requiredCapabilities.contains(INLINESTATS.capabilityName())); - //assumeFalse("INLINESTATS not yet supported in CCS", testCase.requiredCapabilities.contains(JOIN_PLANNING_V1.capabilityName())); - //assumeFalse("INLINESTATS not yet supported in CCS", testCase.requiredCapabilities.contains(INLINESTATS_V11.capabilityName())); + assumeFalse("INLINESTATS not yet supported in CCS", testCase.requiredCapabilities.contains(INLINESTATS.capabilityName())); + assumeFalse("INLINESTATS not yet supported in CCS", testCase.requiredCapabilities.contains(JOIN_PLANNING_V1.capabilityName())); + assumeFalse("INLINESTATS not yet supported in CCS", testCase.requiredCapabilities.contains(INLINESTATS_V11.capabilityName())); if (testCase.requiredCapabilities.contains(JOIN_LOOKUP_V12.capabilityName())) { assumeTrue( "LOOKUP JOIN not yet supported in CCS", From ff6558828b3b34575b1fd504c79b1802b7ae48ea Mon Sep 17 00:00:00 2001 From: Andrei Stefan Date: Tue, 2 Sep 2025 17:10:00 +0300 Subject: [PATCH 06/11] Muted an inlinestats + fork IT test --- muted-tests.yml | 3 +++ .../qa/testFixtures/src/main/resources/inlinestats.csv-spec | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/muted-tests.yml b/muted-tests.yml index 349771fe0a3f5..7857b447bbaf4 100644 --- a/muted-tests.yml +++ b/muted-tests.yml @@ -678,6 +678,9 @@ tests: - class: org.elasticsearch.xpack.logsdb.qa.LogsDbVersusReindexedLogsDbChallengeRestIT method: testEsqlSource issue: https://github.com/elastic/elasticsearch/issues/133520 +- class: org.elasticsearch.xpack.esql.qa.single_node.GenerativeForkIT + method: test {csv-spec:inlinestats.MultiIndexInlinestats OfMultiTypedField} + issue: https://github.com/elastic/elasticsearch/issues/133973 # Examples: # diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/inlinestats.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/inlinestats.csv-spec index e282591693467..561b4f6d8980d 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/inlinestats.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/inlinestats.csv-spec @@ -3084,7 +3084,7 @@ count:long | message:keyword multiIndexInlinestats OfMultiTypedField required_capability: inlinestats_v11 - +// https://github.com/elastic/elasticsearch/issues/133973 FROM apps, apps_short | INLINESTATS s = sum(id::integer) | SORT name, version From 94210ab43cc8b30ba452aa5232decd722705ef47 Mon Sep 17 00:00:00 2001 From: Andrei Stefan Date: Tue, 2 Sep 2025 17:37:28 +0300 Subject: [PATCH 07/11] Fix test after https://github.com/elastic/elasticsearch/pull/133651 merge --- .../src/main/resources/inlinestats.csv-spec | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/inlinestats.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/inlinestats.csv-spec index a5072c389a8d9..eabc2495c2c6c 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/inlinestats.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/inlinestats.csv-spec @@ -2844,7 +2844,7 @@ mc:l | count:l | event_duration:l 6 |2 |3450233 ; -ultiIndexIpStringInlinestats Inline4 +multiIndexIpStringInlinestats Inline4 required_capability: inlinestats_v11 FROM sample_data, sample_data_str @@ -2856,11 +2856,11 @@ FROM sample_data, sample_data_str ; mc:l | count:l | event_duration:l -4 |4 |725448 -4 |4 |1756467 -4 |4 |5033755 -4 |4 |8268153 -1 |1 |1232382 +4 |8 |725448 +4 |8 |1756467 +4 |8 |5033755 +4 |8 |8268153 +1 |2 |1232382 ; multiIndexWhereIpStringInlinestats From aef5aae18d5e81e42e9a313a4171c2e18f146cc1 Mon Sep 17 00:00:00 2001 From: Andrei Stefan Date: Wed, 3 Sep 2025 10:38:01 +0300 Subject: [PATCH 08/11] Address reviews: - added comments to some of the csv-spec tests - de-Ignored few csv-spec tests - renamed (removed the white spaces) csv-spec tests - more tests --- muted-tests.yml | 2 +- .../src/main/resources/inlinestats.csv-spec | 234 +++++++++++++++--- 2 files changed, 207 insertions(+), 29 deletions(-) diff --git a/muted-tests.yml b/muted-tests.yml index b403c4edfe446..85f29b8626b01 100644 --- a/muted-tests.yml +++ b/muted-tests.yml @@ -532,7 +532,7 @@ tests: method: testInferClassificationModel issue: https://github.com/elastic/elasticsearch/issues/133448 - class: org.elasticsearch.xpack.esql.qa.single_node.GenerativeForkIT - method: test {csv-spec:inlinestats.MultiIndexInlinestats OfMultiTypedField} + method: test {csv-spec:inlinestats.MultiIndexInlinestatsOfMultiTypedField} issue: https://github.com/elastic/elasticsearch/issues/133973 # Examples: diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/inlinestats.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/inlinestats.csv-spec index eabc2495c2c6c..698df8ad88624 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/inlinestats.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/inlinestats.csv-spec @@ -2716,7 +2716,7 @@ null |725448 |Connection error |0 null |725448 |Connection error |0 |2023-10-23T13:50:00.000Z ; -multiIndexIpInlinestats NonPushableCount +multiIndexIpInlinestats_NonPushableCount required_capability: inlinestats_v11 FROM sample_data, sample_data_str @@ -2730,7 +2730,7 @@ FROM sample_data, sample_data_str 2023-10-23T13:52:55.015Z|null |8268153 |Connection error|14 ; -multiIndexIpInlinestats NonPushableCountEval +multiIndexIpInlinestats_NonPushableCountEval required_capability: inlinestats_v11 FROM sample_data, sample_data_str @@ -2745,7 +2745,7 @@ FROM sample_data, sample_data_str 2023-10-23T13:52:55.015Z|8268153 |Connection error|172.21.3.15 |14 ; -multiIndexIpInlinestats NonPushableCountWithFilter +multiIndexIpInlinestats_NonPushableCountWithFilter required_capability: inlinestats_v11 FROM sample_data, sample_data_ts_long @@ -2762,7 +2762,7 @@ null |172.21.2.113 |2764889 |Connected to 10.1.0. null |172.21.2.113 |2764889 |Connected to 10.1.0.2|2 |14 ; -multiIndexIpInlinestats PushableCount-Ignore +multiIndexIpInlinestats_PushableCount required_capability: inlinestats_v11 FROM sample_data, sample_data_ts_long @@ -2776,7 +2776,7 @@ count:long 14 ; -multiIndexIpInlinestats PushableCountEval-Ignore +multiIndexIpInlinestats_PushableCountEval required_capability: inlinestats_v11 FROM sample_data, sample_data_ts_long @@ -2791,7 +2791,7 @@ count:long 14 ; -multiIndexIpStringInlinestats Inline2 +multiIndexIpStringInlinestats_Inline2 required_capability: inlinestats_v11 FROM sample_data, sample_data_str @@ -2817,7 +2817,7 @@ count:long | client_ip::ip:ip 2 |172.21.2.162 ; -multiIndexIpStringInlinestats Inline3 +multiIndexIpStringInlinestats_Inline3 required_capability: inlinestats_v11 FROM sample_data, sample_data_str @@ -2844,7 +2844,7 @@ mc:l | count:l | event_duration:l 6 |2 |3450233 ; -multiIndexIpStringInlinestats Inline4 +multiIndexIpStringInlinestats_Inline4 required_capability: inlinestats_v11 FROM sample_data, sample_data_str @@ -2923,7 +2923,7 @@ count:long | @timestamp:date 4 | 2023-10-23T12:00:00.000Z ; -Multi Index millis to nanos stats +multiIndexMillisToNanosStats required_capability: inlinestats_v11 FROM sample_data, sample_data_ts_nanos @@ -2940,7 +2940,7 @@ count:long | @timestamp:date_nanos ; -multiIndexTsLongInlinestats Drop-Ignore +multiIndexTsLongInlinestatsDrop required_capability: inlinestats_v11 FROM sample_data, sample_data_ts_long, sample_data_ts_nanos @@ -2954,7 +2954,7 @@ count:long 3 ; -multiIndexTsLongInlinestats Inline2 +multiIndexTsLongInlinestatsInline2 required_capability: inlinestats_v11 FROM sample_data, sample_data_ts_long, sample_data_ts_nanos @@ -2972,7 +2972,7 @@ null |172.21.3.15 |5033755 |Connection error null |172.21.3.15 |5033755 |Connection error |3 |2023-10-23T13:53:55.832Z ; -multiIndexTsLongInlinestats Inline3 +multiIndexTsLongInlinestatsInline3 required_capability: inlinestats_v11 FROM sample_data, sample_data_ts_long @@ -2987,7 +2987,7 @@ mc:l | count:l 14 | 2 ; -multiIndexTsLongInlinestats Inlinestats +multiIndexTsLongInlinestatsInlinestats required_capability: inlinestats_v11 FROM sample_data, sample_data_ts_long @@ -3072,7 +3072,7 @@ count:long | mess:keyword 1 |1698063 ; -multiIndexTsLongInlinestats Inline-Ignore +multiIndexTsLongInlinestatsInline required_capability: inlinestats_v11 FROM sample_data, sample_data_ts_long @@ -3087,7 +3087,7 @@ count:long | max:date 14 | 2023-10-23T13:55:01.543Z ; -multiIndexTsLongInlinestats InlineDropped-Ignore +multiIndexTsLongInlinestatsInlineDropped required_capability: inlinestats_v11 FROM sample_data, sample_data_ts_long @@ -3150,7 +3150,7 @@ count:long | message:keyword 3 | Connected to 10.1.0.3 ; -multiIndexInlinestats OfMultiTypedField +multiIndexInlinestatsOfMultiTypedField required_capability: inlinestats_v11 // https://github.com/elastic/elasticsearch/issues/133973 FROM apps, apps_short @@ -3272,7 +3272,7 @@ id:integer | name:keyword | count:long 3 |ccccc |2 ; -ImplicitCastingMultiTypedFieldsInlinestats ByNumeric +ImplicitCastingMultiTypedFieldsInlinestats_ByNumeric required_capability: inlinestats_v11 FROM employees, employees_incompatible @@ -3288,7 +3288,7 @@ null |1 |1999-04-30T00:00:00.000Z|1985-02-18T00:00:0 null |1 |1999-04-30T00:00:00.000Z|1985-02-18T00:00:00.000Z ; -ImplicitCastingMultiTypedFieldsInlinestats ByNumericWithFilter +ImplicitCastingMultiTypedFieldsInlinestats_ByNumericWithFilter required_capability: inlinestats_v11 FROM employees, employees_incompatible @@ -3304,7 +3304,7 @@ null |1 |1990-10-20T00:00:00.000Z|1994-05-21T00:00:0 null |1 |1990-10-20T00:00:00.000Z|1994-05-21T00:00:00.000Z ; -ImplicitCastingMultiTypedFieldsInlinestats ByDateNanos +ImplicitCastingMultiTypedFieldsInlinestats_ByDateNanos required_capability: inlinestats_v11 FROM employees, employees_incompatible @@ -3330,7 +3330,7 @@ FROM employees, employees_incompatible 2 |-5.9 |1.8 |1985-10-14T00:00:00.000Z ; -ImplicitCastingMultiTypedFieldsInlinestats ByDateNanosWithFilter +ImplicitCastingMultiTypedFieldsInlinestats_ByDateNanosWithFilter required_capability: inlinestats_v11 FROM employees, employees_incompatible @@ -3354,7 +3354,7 @@ FROM employees, employees_incompatible 2 |3.6 |1.5 |1985-09-17T00:00:00.000Z ; -ImplicitCastingMultiTypedMVFieldsInlinestats MaxMin +ImplicitCastingMultiTypedMVFieldsInlinestats_MaxMin required_capability: inlinestats_v11 FROM date_nanos, date_nanos_union_types @@ -3374,7 +3374,7 @@ warningRegex:java.lang.IllegalArgumentException: milliSeconds \[-1457696696640\] 2023-10-23T13:53:55.832Z |2023-10-23T13:53:55.832987654Z|1698069235832987654|2023-10-23T13:55:01.543123456Z|2023-01-23T13:55:01.543Z ; -ImplicitCastingMultiTypedMVFieldsInlinestats Values +ImplicitCastingMultiTypedMVFieldsInlinestats_Values required_capability: inlinestats_v11 FROM date_nanos, date_nanos_union_types @@ -3394,8 +3394,10 @@ warningRegex:java.lang.IllegalArgumentException: milliSeconds \[-1457696696640\] 2023-10-23T13:53:55.832Z |2023-10-23T13:53:55.832987654Z|1698069235832987654|19 ; -ImplicitCastingMultiTypedDateTruncInlinestats By-Ignore +ImplicitCastingMultiTypedDateTruncInlinestats_By-Ignore required_capability: inlinestats_v11 +// https://github.com/elastic/elasticsearch/issues/133973 +// optimized incorrectly due to missing references [$$emp_no$converted_to$long{f$}# FROM employees, employees_incompatible | KEEP emp_no, hire_date @@ -3412,8 +3414,10 @@ c:long | yr:date_nanos 8 | 1994-01-01T00:00:00.000Z ; -ImplicitCastingMultiTypedDateTruncInlinestats ByWithFilter-Ignore +ImplicitCastingMultiTypedDateTruncInlinestats_ByWithFilter-Ignore required_capability: inlinestats_v11 +// https://github.com/elastic/elasticsearch/issues/133973 +// optimized incorrectly due to missing references [$$emp_no$converted_to$long{f$}# FROM employees, employees_incompatible | KEEP emp_no, hire_date @@ -3430,8 +3434,10 @@ c:long | yr:date_nanos 0 | 1994-01-01T00:00:00.000Z ; -ImplicitCastingMultiTypedDateTruncInlinestats ByWithEval-Ignore +ImplicitCastingMultiTypedDateTruncInlinestats_ByWithEval-Ignore required_capability: inlinestats_v11 +// https://github.com/elastic/elasticsearch/issues/133973 +// optimized incorrectly due to missing references [$$emp_no$converted_to$long{f$}# FROM employees, employees_incompatible | KEEP emp_no, hire_date @@ -3449,8 +3455,10 @@ c:long | yr:date_nanos 8 | 1994-01-01T00:00:00.000Z ; -ImplicitCastingMultiTypedDateTruncInlinestats ByWithEvalWithFilter-Ignore +ImplicitCastingMultiTypedDateTruncInlinestats_ByWithEvalWithFilter-Ignore required_capability: inlinestats_v11 +// https://github.com/elastic/elasticsearch/issues/133973 +// optimized incorrectly due to missing references [$$emp_no$converted_to$long{f$}# FROM employees, employees_incompatible | KEEP emp_no, hire_date @@ -3537,7 +3545,7 @@ null |1995-03-13T00:00:00.000Z|10 |1995-01-01T00:00:00. null |1995-03-13T00:00:00.000Z|10 |1995-01-01T00:00:00.000Z ; -ImplicitCastingMultiTypedBucketDateNanosInBothInlinestats AndBy +ImplicitCastingMultiTypedBucketDateNanos_In_BothInlinestats_AndBy required_capability: inlinestats_v11 FROM employees, employees_incompatible @@ -3560,7 +3568,7 @@ null |1995-03-13T00:00:00.000Z|10 |1996-01-01T00:00:00. null |1995-03-13T00:00:00.000Z|10 |1996-01-01T00:00:00.000Z|1995-01-01T00:00:00.000Z ; -ImplicitCastingMultiTypedBucketDateNanosInBothInlinestats AndByWithAlias +ImplicitCastingMultiTypedBucketDateNanos_In_BothInlinestats_AndByWithAlias required_capability: inlinestats_v11 FROM employees, employees_incompatible @@ -3582,3 +3590,173 @@ null |1995-12-15T00:00:00.000Z|10 |1996-01-01T00:00:00. null |1995-08-22T00:00:00.000Z|10 |1996-01-01T00:00:00.000Z|1995-01-01T00:00:00.000Z null |1995-08-22T00:00:00.000Z|10 |1996-01-01T00:00:00.000Z|1995-01-01T00:00:00.000Z ; + +multiInlinestatsWithUnionTypes1 +required_capability: inlinestats_v11 + +FROM sample_data, sample_data_str +| EVAL one_ip = client_ip::ip +| INLINESTATS count1=count(client_ip::ip), count2=count(one_ip), max(one_ip), min(client_ip::ip) +| SORT @timestamp +| LIMIT 3 +; + + @timestamp:date |client_ip:unsupported|event_duration:long| message:keyword | one_ip:ip | count1:long| count2:long|max(one_ip):ip |min(client_ip::ip):ip +2023-10-23T12:15:03.360Z|null |3450233 |Connected to 10.1.0.3|172.21.2.162 |14 |14 |172.21.3.15 |172.21.0.5 +2023-10-23T12:15:03.360Z|null |3450233 |Connected to 10.1.0.3|172.21.2.162 |14 |14 |172.21.3.15 |172.21.0.5 +2023-10-23T12:27:28.948Z|null |2764889 |Connected to 10.1.0.2|172.21.2.113 |14 |14 |172.21.3.15 |172.21.0.5 +; + +multiInlinestatsWithUnionTypes2 +required_capability: inlinestats_v11 + +FROM sample_data, sample_data_str +| EVAL one_ip = client_ip::ip +| INLINESTATS count1=count(client_ip::ip), count2=count(one_ip), max=max(one_ip), min=min(client_ip::ip) +| RENAME one_ip AS client_ip +| INLINESTATS max(event_duration) BY client_ip +| SORT @timestamp +| LIMIT 5 +; + +@timestamp:date | event_duration:long | message:keyword | count1:long | count2:long | max:ip | min:ip | max(event_duration):long| client_ip:ip +2023-10-23T12:15:03.360Z| 3450233 | Connected to 10.1.0.3 | 14 | 14 | 172.21.3.15 | 172.21.0.5 | 3450233 | 172.21.2.162 +2023-10-23T12:15:03.360Z| 3450233 | Connected to 10.1.0.3 | 14 | 14 | 172.21.3.15 | 172.21.0.5 | 3450233 | 172.21.2.162 +2023-10-23T12:27:28.948Z| 2764889 | Connected to 10.1.0.2 | 14 | 14 | 172.21.3.15 | 172.21.0.5 | 2764889 | 172.21.2.113 +2023-10-23T12:27:28.948Z| 2764889 | Connected to 10.1.0.2 | 14 | 14 | 172.21.3.15 | 172.21.0.5 | 2764889 | 172.21.2.113 +2023-10-23T13:33:34.937Z| 1232382 | Disconnected | 14 | 14 | 172.21.3.15 | 172.21.0.5 | 1232382 | 172.21.0.5 +; + +multiInlinestatsWithUnionTypes3 +required_capability: inlinestats_v11 + +FROM sample_data, sample_data_str +| EVAL one_ip = client_ip::ip +| INLINESTATS avg_duration = AVG(event_duration) BY one_ip +| RENAME one_ip AS client_ip +| WHERE CIDR_MATCH(client_ip, "172.21.2.0/24") +| INLINESTATS count1=count(client_ip::ip), count2=count(client_ip), max=max(client_ip), min=min(client_ip::ip) +| INLINESTATS avg_total_duration = AVG(event_duration) +| SORT @timestamp +; + +@timestamp:date | event_duration:long| message:keyword | avg_duration:double| client_ip:ip | count1:long | count2:long | max:ip | min:ip | avg_total_duration:double +2023-10-23T12:15:03.360Z | 3450233 | Connected to 10.1.0.3 | 3450233.0 | 172.21.2.162 | 4 | 4 | 172.21.2.162 | 172.21.2.113 | 3107561.0 +2023-10-23T12:15:03.360Z | 3450233 | Connected to 10.1.0.3 | 3450233.0 | 172.21.2.162 | 4 | 4 | 172.21.2.162 | 172.21.2.113 | 3107561.0 +2023-10-23T12:27:28.948Z | 2764889 | Connected to 10.1.0.2 | 2764889.0 | 172.21.2.113 | 4 | 4 | 172.21.2.162 | 172.21.2.113 | 3107561.0 +2023-10-23T12:27:28.948Z | 2764889 | Connected to 10.1.0.2 | 2764889.0 | 172.21.2.113 | 4 | 4 | 172.21.2.162 | 172.21.2.113 | 3107561.0 +; + +unionTypeDroppedBeforeInlinestats +required_capability: inlinestats_v11 + +FROM sample_data, sample_data_str +| EVAL one_ip = client_ip::ip +| DROP one_ip +| INLINESTATS avg(event_duration) BY client_ip::ip +| SORT @timestamp +; + +@timestamp:date | client_ip:unsupported | event_duration:long| message:keyword | avg(event_duration):double| client_ip::ip:ip +2023-10-23T12:15:03.360Z | null | 3450233 | Connected to 10.1.0.3 | 3450233.0 | 172.21.2.162 +2023-10-23T12:15:03.360Z | null | 3450233 | Connected to 10.1.0.3 | 3450233.0 | 172.21.2.162 +2023-10-23T12:27:28.948Z | null | 2764889 | Connected to 10.1.0.2 | 2764889.0 | 172.21.2.113 +2023-10-23T12:27:28.948Z | null | 2764889 | Connected to 10.1.0.2 | 2764889.0 | 172.21.2.113 +2023-10-23T13:33:34.937Z | null | 1232382 | Disconnected | 1232382.0 | 172.21.0.5 +2023-10-23T13:33:34.937Z | null | 1232382 | Disconnected | 1232382.0 | 172.21.0.5 +2023-10-23T13:51:54.732Z | null | 725448 | Connection error | 3945955.75 | 172.21.3.15 +2023-10-23T13:51:54.732Z | null | 725448 | Connection error | 3945955.75 | 172.21.3.15 +2023-10-23T13:52:55.015Z | null | 8268153 | Connection error | 3945955.75 | 172.21.3.15 +2023-10-23T13:52:55.015Z | null | 8268153 | Connection error | 3945955.75 | 172.21.3.15 +2023-10-23T13:53:55.832Z | null | 5033755 | Connection error | 3945955.75 | 172.21.3.15 +2023-10-23T13:53:55.832Z | null | 5033755 | Connection error | 3945955.75 | 172.21.3.15 +2023-10-23T13:55:01.543Z | null | 1756467 | Connected to 10.1.0.1 | 3945955.75 | 172.21.3.15 +2023-10-23T13:55:01.543Z | null | 1756467 | Connected to 10.1.0.1 | 3945955.75 | 172.21.3.15 +; + +inlinestatsWithUnionTypesAs_InlinestatsCondition +required_capability: inlinestats_v11 + +FROM sample_data, sample_data_str +| EVAL temp_ip = client_ip::ip +| EVAL ip1 = CIDR_MATCH(temp_ip, "172.21.2.0/24"), ip2 = CIDR_MATCH(temp_ip, "172.21.0.0/24"), ip3 = CIDR_MATCH(temp_ip, "172.21.3.0/24") +| EVAL client_ip = temp_ip +| INLINESTATS avg_ip1 = avg(event_duration) WHERE ip1 +| DROP client_ip +| INLINESTATS avg_ip2 = avg(event_duration) WHERE ip2, avg_ip3 = avg(event_duration) WHERE ip3 +| SORT @timestamp +| DROP @timestamp +; + +event_duration:long | message:keyword | temp_ip:ip | ip1:boolean | ip2:boolean | ip3:boolean | avg_ip1:double | avg_ip2:double | avg_ip3:double +3450233 | Connected to 10.1.0.3 | 172.21.2.162 | true | false | false | 3107561.0 | 1232382.0 | 3945955.75 +3450233 | Connected to 10.1.0.3 | 172.21.2.162 | true | false | false | 3107561.0 | 1232382.0 | 3945955.75 +2764889 | Connected to 10.1.0.2 | 172.21.2.113 | true | false | false | 3107561.0 | 1232382.0 | 3945955.75 +2764889 | Connected to 10.1.0.2 | 172.21.2.113 | true | false | false | 3107561.0 | 1232382.0 | 3945955.75 +1232382 | Disconnected | 172.21.0.5 | false | true | false | 3107561.0 | 1232382.0 | 3945955.75 +1232382 | Disconnected | 172.21.0.5 | false | true | false | 3107561.0 | 1232382.0 | 3945955.75 +725448 | Connection error | 172.21.3.15 | false | false | true | 3107561.0 | 1232382.0 | 3945955.75 +725448 | Connection error | 172.21.3.15 | false | false | true | 3107561.0 | 1232382.0 | 3945955.75 +8268153 | Connection error | 172.21.3.15 | false | false | true | 3107561.0 | 1232382.0 | 3945955.75 +8268153 | Connection error | 172.21.3.15 | false | false | true | 3107561.0 | 1232382.0 | 3945955.75 +5033755 | Connection error | 172.21.3.15 | false | false | true | 3107561.0 | 1232382.0 | 3945955.75 +5033755 | Connection error | 172.21.3.15 | false | false | true | 3107561.0 | 1232382.0 | 3945955.75 +1756467 | Connected to 10.1.0.1 | 172.21.3.15 | false | false | true | 3107561.0 | 1232382.0 | 3945955.75 +1756467 | Connected to 10.1.0.1 | 172.21.3.15 | false | false | true | 3107561.0 | 1232382.0 | 3945955.75 +; + +inlinestatsWithOverwrittenUnionType +required_capability: inlinestats_v11 + +FROM sample_data, sample_data_str +| EVAL client_ip = client_ip::ip +| EVAL client_ip = 123 +| INLINESTATS a = avg(event_duration) BY client_ip::integer +| SORT @timestamp +; + +@timestamp:date | event_duration:long | message:keyword | client_ip:integer | a:double | client_ip::integer:integer +2023-10-23T12:15:03.360Z | 3450233 | Connected to 10.1.0.3 | 123 | 3318761.0 | 123 +2023-10-23T12:15:03.360Z | 3450233 | Connected to 10.1.0.3 | 123 | 3318761.0 | 123 +2023-10-23T12:27:28.948Z | 2764889 | Connected to 10.1.0.2 | 123 | 3318761.0 | 123 +2023-10-23T12:27:28.948Z | 2764889 | Connected to 10.1.0.2 | 123 | 3318761.0 | 123 +2023-10-23T13:33:34.937Z | 1232382 | Disconnected | 123 | 3318761.0 | 123 +2023-10-23T13:33:34.937Z | 1232382 | Disconnected | 123 | 3318761.0 | 123 +2023-10-23T13:51:54.732Z | 725448 | Connection error | 123 | 3318761.0 | 123 +2023-10-23T13:51:54.732Z | 725448 | Connection error | 123 | 3318761.0 | 123 +2023-10-23T13:52:55.015Z | 8268153 | Connection error | 123 | 3318761.0 | 123 +2023-10-23T13:52:55.015Z | 8268153 | Connection error | 123 | 3318761.0 | 123 +2023-10-23T13:53:55.832Z | 5033755 | Connection error | 123 | 3318761.0 | 123 +2023-10-23T13:53:55.832Z | 5033755 | Connection error | 123 | 3318761.0 | 123 +2023-10-23T13:55:01.543Z | 1756467 | Connected to 10.1.0.1 | 123 | 3318761.0 | 123 +2023-10-23T13:55:01.543Z | 1756467 | Connected to 10.1.0.1 | 123 | 3318761.0 | 123 +; + +overwriteAndDropUnionTypeBeforeInlinestats +required_capability: inlinestats_v11 + +FROM sample_data, sample_data_str +| EVAL temp = client_ip::string +| EVAL client_ip = temp +| DROP temp +| INLINESTATS avg(event_duration) BY client_ip +| SORT @timestamp +| DROP @timestamp +; + +event_duration:long| message:keyword | avg(event_duration):double| client_ip:string +3450233 | Connected to 10.1.0.3 | 3450233.0 | 172.21.2.162 +3450233 | Connected to 10.1.0.3 | 3450233.0 | 172.21.2.162 +2764889 | Connected to 10.1.0.2 | 2764889.0 | 172.21.2.113 +2764889 | Connected to 10.1.0.2 | 2764889.0 | 172.21.2.113 +1232382 | Disconnected | 1232382.0 | 172.21.0.5 +1232382 | Disconnected | 1232382.0 | 172.21.0.5 +725448 | Connection error | 3945955.75 | 172.21.3.15 +725448 | Connection error | 3945955.75 | 172.21.3.15 +8268153 | Connection error | 3945955.75 | 172.21.3.15 +8268153 | Connection error | 3945955.75 | 172.21.3.15 +5033755 | Connection error | 3945955.75 | 172.21.3.15 +5033755 | Connection error | 3945955.75 | 172.21.3.15 +1756467 | Connected to 10.1.0.1 | 3945955.75 | 172.21.3.15 +1756467 | Connected to 10.1.0.1 | 3945955.75 | 172.21.3.15 +; From c20da1787a5d359f8fb13e4c524e75392fb9fd28 Mon Sep 17 00:00:00 2001 From: Andrei Stefan Date: Thu, 4 Sep 2025 15:51:18 +0300 Subject: [PATCH 09/11] make the test more deterministic --- .../src/main/resources/inlinestats.csv-spec | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/inlinestats.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/inlinestats.csv-spec index 698df8ad88624..cc80009aede1a 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/inlinestats.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/inlinestats.csv-spec @@ -2848,6 +2848,8 @@ multiIndexIpStringInlinestats_Inline4 required_capability: inlinestats_v11 FROM sample_data, sample_data_str +| SORT @timestamp +| LIMIT 1000 | INLINESTATS count=count(*) BY client_ip::ip | LIMIT 5 | INLINESTATS mc=count(count) BY count @@ -2856,11 +2858,11 @@ FROM sample_data, sample_data_str ; mc:l | count:l | event_duration:l -4 |8 |725448 -4 |8 |1756467 -4 |8 |5033755 -4 |8 |8268153 -1 |2 |1232382 +5 |2 |1232382 +5 |2 |2764889 +5 |2 |2764889 +5 |2 |3450233 +5 |2 |3450233 ; multiIndexWhereIpStringInlinestats From 73caf5df8943a658f872556522b9167f7b0f45a4 Mon Sep 17 00:00:00 2001 From: Andrei Stefan Date: Thu, 4 Sep 2025 17:24:55 +0300 Subject: [PATCH 10/11] more deterministic tests --- .../qa/testFixtures/src/main/resources/inlinestats.csv-spec | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/inlinestats.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/inlinestats.csv-spec index cc80009aede1a..a73fa527c671e 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/inlinestats.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/inlinestats.csv-spec @@ -3037,8 +3037,8 @@ required_capability: inlinestats_v11 FROM sample_data, sample_data_ts_long | EVAL hour = LEFT(TO_STRING(TO_DATETIME(@timestamp)), 13) | INLINESTATS count=count(*) BY hour -| SORT count DESC, hour ASC | STATS count_distinct(count) BY count, hour +| SORT count DESC, hour ASC | KEEP count, hour ; From 98ae6b87970fa31972b1f7dab5711419f6e0bc8e Mon Sep 17 00:00:00 2001 From: Andrei Stefan Date: Thu, 4 Sep 2025 18:25:32 +0300 Subject: [PATCH 11/11] More tests adjustments --- .../qa/testFixtures/src/main/resources/inlinestats.csv-spec | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/inlinestats.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/inlinestats.csv-spec index a73fa527c671e..ae0eba15fed64 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/inlinestats.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/inlinestats.csv-spec @@ -2915,9 +2915,9 @@ required_capability: inlinestats_v11 FROM sample_data, sample_data_ts_nanos | EVAL @timestamp = DATE_TRUNC(1 hour, TO_DATETIME(@timestamp)) | INLINESTATS count=count(*) BY @timestamp -| SORT count DESC, @timestamp ASC | KEEP count, @timestamp | STATS count=count(*) BY @timestamp +| SORT count DESC, @timestamp ASC ; count:long | @timestamp:date @@ -2931,9 +2931,9 @@ required_capability: inlinestats_v11 FROM sample_data, sample_data_ts_nanos | EVAL @timestamp = DATE_TRUNC(1 hour, TO_DATE_NANOS(@timestamp)) | INLINESTATS count=count(*) BY @timestamp -| SORT count DESC, @timestamp ASC | KEEP count, @timestamp | STATS count=count(*) BY @timestamp +| SORT count DESC, @timestamp ASC ; count:long | @timestamp:date_nanos