PromQL Support in ESQL (elastic#137988)

* Introduce PromQL command

Add PromQL support in ESQL under the PROMQL command.
This commit includes the grammar, associated logical plan and basic
rules for performing translation, from filtering to stats transpilation.

* Grammar
Stand-alone grammar definition and parser, currently 'embedded' inside
ESQL (the current scenario).
Grammar include to pass the valid tests (~200 queries) from Prometheus
3.5; the invalid ones are disabled for now since the parser needs to
validate the syntax using

* Transpilation
Implement basic PromQL plan translation:
 Selector → Filter with label matcher conditions
 RangeSelector → Filter + Bucket for time bucketing
 WithinSeriesAggregate → ESQL function over selector
 AcrossSeriesAggregate → TimeSeriesAggregate with groupings

---------

Co-authored-by: Felix Barnsteiner <[email protected]>

Author: 2 people

.gitattributes

@@ -4,12 +4,13 @@ CHANGELOG.asciidoc  merge=union
 # Windows
 build-tools-internal/src/test/resources/org/elasticsearch/gradle/internal/release/*.asciidoc text eol=lf
 
+# ESQL parsing and source generated related assets
 x-pack/plugin/esql/compute/src/main/generated/** linguist-generated=true
 x-pack/plugin/esql/compute/src/main/generated-src/** linguist-generated=true
 x-pack/plugin/esql/src/main/antlr/*.tokens linguist-generated=true
 x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/*.interp linguist-generated=true
-x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseLexer*.java linguist-generated=true
-x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParser*.java linguist-generated=true
+x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/*BaseLexer*.java linguist-generated=true
+x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/*BaseParser*.java linguist-generated=true
 x-pack/plugin/esql/src/main/generated/** linguist-generated=true
 x-pack/plugin/esql/src/main/generated-src/** linguist-generated=true
 

build-tools-internal/src/main/resources/checkstyle_suppressions.xml

@@ -12,7 +12,7 @@
   <suppress files="modules[/\\]lang-painless[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]painless[/\\]antlr[/\\]SuggestLexer\.java" checks="." />
   <suppress files="plugin[/\\]sql[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]xpack[/\\]sql[/\\]parser[/\\]SqlBase(Base(Listener|Visitor)|Lexer|Listener|Parser|Visitor).java" checks="." />
   <suppress files="plugin[/\\]eql[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]xpack[/\\]eql[/\\]parser[/\\]EqlBase(Base(Listener|Visitor)|Lexer|Listener|Parser|Visitor).java" checks="." />
-  <suppress files="plugin[/\\]esql[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]xpack[/\\]esql[/\\]parser[/\\]EsqlBase(Parser|Lexer).*.java" checks="." />
+  <suppress files="plugin[/\\]esql[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]xpack[/\\]esql[/\\]parser[/\\](EsqlBase|PromqlBase)(Parser|Lexer).*.java" checks="." />
   <suppress files="x-pack[/\\]plugin[/\\]otel-data[/\\]build[/\\]generated[/\\]sources[/\\]" checks="." />
 
   <!-- JNA requires the no-argument constructor on JNAKernel32Library.SizeT to be public-->

x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/Literal.java

@@ -22,6 +22,7 @@
 
 import java.io.IOException;
 import java.time.Duration;
+import java.time.Instant;
 import java.util.Collection;
 import java.util.Objects;
 
@@ -217,6 +218,10 @@ public static Literal timeDuration(Source source, Duration literal) {
         return new Literal(source, literal, DataType.TIME_DURATION);
     }
 
+    public static Literal dateTime(Source source, Instant literal) {
+        return new Literal(source, literal, DataType.DATETIME);
+    }
+
     public static Literal integer(Source source, Integer literal) {
         return new Literal(source, literal, INTEGER);
     }
@@ -229,6 +234,10 @@ public static Literal fromLong(Source source, Long literal) {
         return new Literal(source, literal, LONG);
     }
 
+    public static Expression fromBoolean(Source source, Boolean literal) {
+        return new Literal(source, literal, DataType.BOOLEAN);
+    }
+
     private static BytesRef longAsWKB(DataType dataType, long encoded) {
         return dataType == GEO_POINT ? GEO.longAsWkb(encoded) : CARTESIAN.longAsWkb(encoded);
     }

x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/predicate/operator/arithmetic/Arithmetics.java

@@ -148,7 +148,7 @@ public static Number mod(Number l, Number r) {
         return l.intValue() % r.intValue();
     }
 
-    static Number negate(Number n) {
+    public static Number negate(Number n) {
         if (n == null) {
             return null;
         }

x-pack/plugin/esql/build.gradle

@@ -332,6 +332,8 @@ pluginManager.withPlugin('com.diffplug.spotless') {
       // for some reason "${outputPath}/EsqlBaseParser*.java" does not match the same files...
       targetExclude "src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseLexer*.java",
         "src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParser*.java",
+        "src/main/java/org/elasticsearch/xpack/esql/parser/PromqlBaseLexer*.java",
+        "src/main/java/org/elasticsearch/xpack/esql/parser/PromqlBaseParser*.java",
         "src/main/generated/**/*.java",
         "src/main/generated-src/generated/**/*.java"
       toggleOffOn('begin generated imports', 'end generated imports')
@@ -345,6 +347,7 @@ tasks.register("cleanGenerated", Delete) {
   }
   delete fileTree(outputPath) {
     include 'EsqlBase*.java'
+    include 'PromqlBase*.java'
   }
 }
 
@@ -384,52 +387,93 @@ tasks.register("regenParser", JavaExec) {
     "${file(grammarPath)}/EsqlBaseParser.g4"
 }
 
+tasks.register("regenPromqlLexer", JavaExec) {
+  dependsOn "cleanGenerated"
+  mainClass = 'org.antlr.v4.Tool'
+  classpath = configurations.regenerate
+  systemProperty 'file.encoding', 'UTF-8'
+  systemProperty 'user.language', 'en'
+  systemProperty 'user.country', 'US'
+  systemProperty 'user.variant', ''
+  args '-Werror',
+    '-package', 'org.elasticsearch.xpack.esql.parser',
+    '-listener',
+    '-visitor',
+    '-lib', "${file(grammarPath)}",
+    '-o', outputPath,
+    "${file(grammarPath)}/PromqlBaseLexer.g4"
+}
+
+tasks.register("regenPromqlParser", JavaExec) {
+  dependsOn "cleanGenerated"
+  dependsOn "regenPromqlLexer"
+  mainClass = 'org.antlr.v4.Tool'
+  classpath = configurations.regenerate
+  systemProperty 'file.encoding', 'UTF-8'
+  systemProperty 'user.language', 'en'
+  systemProperty 'user.country', 'US'
+  systemProperty 'user.variant', ''
+  args '-Werror',
+    '-package', 'org.elasticsearch.xpack.esql.parser',
+    '-listener',
+    '-visitor',
+    '-lib', outputPath,
+    '-lib', "${file(grammarPath)}",
+    '-o', outputPath,
+    "${file(grammarPath)}/PromqlBaseParser.g4"
+}
+
 tasks.register("regen") {
   dependsOn "regenParser"
+  dependsOn "regenPromqlParser"
   doLast {
     // moves token files to grammar directory for use with IDE's
     ant.move(file: "${outputPath}/EsqlBaseLexer.tokens", toDir: grammarPath)
     ant.move(file: "${outputPath}/EsqlBaseParser.tokens", toDir: grammarPath)
+    ant.move(file: "${outputPath}/PromqlBaseLexer.tokens", toDir: grammarPath)
+    ant.move(file: "${outputPath}/PromqlBaseParser.tokens", toDir: grammarPath)
+
     // make the generated classes package private
     ant.replaceregexp(
-      match: 'public ((interface|class) \\QEsqlBase(Parser|Lexer)\\E\\w+)',
+      match: 'public ((interface|class) \\Q(Es|Prom)qlBase(Parser|Lexer)\\E\\w+)',
       replace: '\\1',
       encoding: 'UTF-8'
     ) {
-      fileset(dir: outputPath, includes: 'EsqlBase*.java')
+      fileset(dir: outputPath, includes: '*qlBase*.java')
     }
     // nuke timestamps/filenames in generated files
     ant.replaceregexp(
       match: '\\Q// Generated from \\E.*',
       replace: '\\/\\/ ANTLR GENERATED CODE: DO NOT EDIT',
       encoding: 'UTF-8'
     ) {
-      fileset(dir: outputPath, includes: 'EsqlBase*.java')
+      fileset(dir: outputPath, includes: '*qlBase*.java')
     }
     // remove tabs in antlr generated files
     ant.replaceregexp(match: '\t', flags: 'g', replace: '  ', encoding: 'UTF-8') {
-      fileset(dir: outputPath, includes: 'EsqlBase*.java')
+      fileset(dir: outputPath, includes: '*qlBase*.java')
     }
     // suppress this-escape warnings on EsqlBaseLexer
     ant.replaceregexp(
-      match: 'public EsqlBaseLexer',
-      replace: '@SuppressWarnings("this-escape")${line.separator}  public EsqlBaseLexer',
+      match: 'public ((Es|Prom)qlBaseLexer)',
+      replace: '@SuppressWarnings("this-escape")${line.separator}  public \\1',
       encoding: 'UTF-8'
     ) {
-      fileset(dir: outputPath, includes: 'EsqlBaseLexer.java')
+      fileset(dir: outputPath, includes: '*qlBaseLexer.java')
     }
+
     // suppress this-escape warnings on all internal EsqlBaseParser class constructores
     ant.replaceregexp(
       match: '([ ]+)public ([A-Z][a-z]+[a-z,A-Z]+\\()',
       flags: 'g',
       replace: '\\1@SuppressWarnings("this-escape")${line.separator}\\1public \\2',
       encoding: 'UTF-8'
     ) {
-      fileset(dir: outputPath, includes: 'EsqlBaseParser.java')
+      fileset(dir: outputPath, includes: '*qlBaseParser.java')
     }
     // fix line endings
     ant.fixcrlf(srcdir: outputPath, eol: 'lf') {
-      patternset(includes: 'EsqlBase*.java')
+      patternset(includes: '*qlBase*.java')
     }
   }
 }

x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/generative/GenerativeForkRestTest.java

@@ -60,6 +60,8 @@ protected void shouldSkipTest(String testName) throws IOException {
             testCase.requiredCapabilities.contains(SUBQUERY_IN_FROM_COMMAND.capabilityName())
         );
 
+        assumeFalse("Tests using PROMQL are not supported for now", testCase.requiredCapabilities.contains(PROMQL_V0.capabilityName()));
+
         assumeTrue("Cluster needs to support FORK", hasCapabilities(adminClient(), List.of(FORK_V9.capabilityName())));
     }
 }

x-pack/plugin/esql/qa/testFixtures/src/main/resources/k8s-timeseries-avg-over-time.csv-spec

@@ -17,6 +17,25 @@ cost:double       | time_bucket:datetime
 36.375            | 2024-05-10T00:11:00.000Z
 ;
 
+avg_over_time_of_double_no_grouping_single_bucket
+required_capability: ts_command_v0
+required_capability: tbucket
+TS k8s
+| STATS cost=sum(avg_over_time(network.cost)) BY time_bucket = tbucket(1h);
+
+cost:double       | time_bucket:datetime
+56.32254035241995 | 2024-05-10T00:00:00.000Z
+;
+
+avg_over_time_of_double_no_grouping_single_bucket_promql
+required_capability: promql_v0
+TS k8s
+| PROMQL step 1h (sum(avg_over_time(network.cost[1h])));
+
+sum(avg_over_time(network.cost[1h])):double | TBUCKET:date
+56.32254035241995                           | 2024-05-10T00:00:00.000Z
+;
+
 avg_over_time_of_integer
 required_capability: ts_command_v0
 TS k8s | STATS clients = avg(avg_over_time(network.eth0.currently_connected_clients)) BY time_bucket = bucket(@timestamp,1minute) | SORT time_bucket | LIMIT 10;

x-pack/plugin/esql/qa/testFixtures/src/main/resources/k8s-timeseries.csv-spec

@@ -132,6 +132,25 @@ max(rate(network.total_bytes_in)): double | time_bucket:date
 23.702205882352942                        | 2024-05-10T00:15:00.000Z
 ;
 
+oneRateWithSingleTBucket
+required_capability: ts_command_v0
+required_capability: tbucket
+TS k8s
+| STATS max(rate(network.total_bytes_in)) BY time_bucket = tbucket(1h);
+
+max(rate(network.total_bytes_in)): double | time_bucket:date
+4.379885057471264                         | 2024-05-10T00:00:00.000Z
+;
+
+oneRateWithSingleTBucketPromql
+required_capability: promql_v0
+TS k8s
+| PROMQL step 1h (max(rate(network.total_bytes_in[1h])));
+
+max(rate(network.total_bytes_in[1h])):double | TBUCKET:date
+4.379885057471264                            | 2024-05-10T00:00:00.000Z
+;
+
 twoRatesWithBucket
 required_capability: ts_command_v0
 TS k8s | STATS max(rate(network.total_bytes_in)), sum(rate(network.total_bytes_in)) BY time_bucket = bucket(@timestamp,5minute) | SORT time_bucket DESC | LIMIT 3;