ESQL: add date_diff function (#104118)

Same as #103208

Fixes #101942

We had to revert it after a Checkstyle failure (strange it didn't pop up
in the CI before merging)

Author: luigidellaquila

docs/changelog/104118.yaml

@@ -0,0 +1,6 @@
+pr: 104118
+summary: "ESQL: add `date_diff` function"
+area: ES|QL
+type: enhancement
+issues:
+ - 101942

docs/reference/esql/functions/date_diff.asciidoc

@@ -0,0 +1,37 @@
+[discrete]
+[[esql-date_diff]]
+=== `DATE_DIFF`
+Subtract the second argument from the third argument and return their difference in multiples of the unit specified in the first argument.
+If the second argument (start) is greater than the third argument (end), then negative values are returned.
+
+[cols="^,^"]
+|===
+2+h|Datetime difference units
+
+s|unit
+s|abbreviations
+
+| year        | years, yy, yyyy
+| quarter     | quarters, qq, q
+| month       | months, mm, m
+| dayofyear   | dy, y
+| day         | days, dd, d
+| week        | weeks, wk, ww
+| weekday     | weekdays, dw
+| hour        | hours, hh
+| minute      | minutes, mi, n
+| second      | seconds, ss, s
+| millisecond | milliseconds, ms
+| microsecond | microseconds, mcs
+| nanosecond  | nanoseconds, ns
+|===
+
+[source.merge.styled,esql]
+----
+include::{esql-specs}/docs.csv-spec[tag=dateDiff]
+----
+[%header.monospaced.styled,format=dsv,separator=|]
+|===
+include::{esql-specs}/docs.csv-spec[tag=dateDiff-result]
+|===
+

docs/reference/esql/functions/signature/date_diff.svg

@@ -0,0 +1,6 @@
+[%header.monospaced.styled,format=dsv,separator=|]
+|===
+unit | startTimestamp | endTimestamp | result
+keyword | datetime | datetime | integer
+text | datetime | datetime | integer
+|===

docs/reference/esql/functions/types/date_diff.asciidoc

@@ -337,6 +337,67 @@ AVG(salary):double | bucket:date
 // end::auto_bucket_in_agg-result[]
 ;
 
+evalDateDiffInNanoAndMicroAndMilliSeconds#[skip:-8.12.99, reason:date_diff added in 8.13]
+ROW date1=to_datetime("2023-12-02T11:00:00.000Z"), date2=to_datetime("2023-12-02T11:00:00.001Z") 
+| EVAL dd_ns1=date_diff("nanoseconds", date1, date2), dd_ns2=date_diff("ns", date1, date2)
+| EVAL dd_mcs1=date_diff("microseconds", date1, date2), dd_mcs2=date_diff("mcs", date1, date2)
+| EVAL dd_ms1=date_diff("milliseconds", date1, date2), dd_ms2=date_diff("ms", date1, date2)
+| keep dd_ns1, dd_ns2, dd_mcs1, dd_mcs2, dd_ms1, dd_ms2
+;
+
+dd_ns1:integer  | dd_ns2:integer    | dd_mcs1:integer   | dd_mcs2:integer   | dd_ms1:integer    | dd_ms2:integer
+1000000         | 1000000           | 1000              | 1000              | 1                 | 1                         
+;
+
+evalDateDiffInSecondsAndMinutesAndHours#[skip:-8.12.99, reason:date_diff added in 8.13]
+ROW date1=to_datetime("2023-12-02T11:00:00.000Z"), date2=to_datetime("2023-12-02T12:00:00.000Z") 
+| EVAL dd_s1=date_diff("seconds", date1, date2), dd_s2=date_diff("ss", date1, date2), dd_s3=date_diff("s", date1, date2)
+| EVAL dd_m1=date_diff("minutes", date1, date2), dd_m2=date_diff("mi", date1, date2), dd_m3=date_diff("n", date1, date2)
+| EVAL dd_h1=date_diff("hours", date1, date2), dd_h2=date_diff("hh", date1, date2)
+| keep dd_s1, dd_s2, dd_s3, dd_m1, dd_m2, dd_m3, dd_h1, dd_h2
+;
+
+dd_s1:integer   | dd_s2:integer | dd_s3:integer | dd_m1:integer | dd_m2:integer | dd_m3:integer | dd_h1:integer | dd_h2:integer
+3600            | 3600          | 3600          | 60            | 60            | 60            | 1             | 1                         
+;
+
+evalDateDiffInDaysAndWeeks#[skip:-8.12.99, reason:date_diff added in 8.13]
+ROW date1=to_datetime("2023-12-02T11:00:00.000Z"), date2=to_datetime("2023-12-24T11:00:00.000Z") 
+| EVAL dd_wd1=date_diff("weekdays", date1, date2), dd_wd2=date_diff("dw", date1, date2)
+| EVAL dd_w1=date_diff("weeks", date1, date2), dd_w2=date_diff("wk", date1, date2), dd_w3=date_diff("ww", date1, date2)
+| EVAL dd_d1=date_diff("dy", date1, date2), dd_d2=date_diff("y", date1, date2)
+| EVAL dd_dy1=date_diff("days", date1, date2), dd_dy2=date_diff("dd", date1, date2), dd_dy3=date_diff("d", date1, date2)
+| keep dd_wd1, dd_wd2, dd_w1, dd_w2, dd_w3, dd_d1, dd_d2, dd_dy1, dd_dy2, dd_dy3
+;
+
+dd_wd1:integer  | dd_wd2:integer    | dd_w1:integer | dd_w2:integer | dd_w3:integer | dd_d1:integer | dd_d2:integer | dd_dy1:integer    | dd_dy2:integer    | dd_dy3:integer
+22              | 22                | 3             | 3             | 3             | 22            | 22            | 22                | 22                | 22                     
+;
+
+evalDateDiffInMonthsAndQuartersAndYears#[skip:-8.12.99, reason:date_diff added in 8.13]
+ROW date1=to_datetime("2023-12-02T11:00:00.000Z"), date2=to_datetime("2024-12-24T11:00:00.000Z") 
+| EVAL dd_m1=date_diff("months", date1, date2), dd_m2=date_diff("mm", date1, date2), dd_m3=date_diff("m", date1, date2)
+| EVAL dd_q1=date_diff("quarters", date1, date2), dd_q2=date_diff("qq", date1, date2), dd_q3=date_diff("q", date1, date2)
+| EVAL dd_y1=date_diff("years", date1, date2), dd_y2=date_diff("yyyy", date1, date2), dd_y3=date_diff("yy", date1, date2)
+| keep dd_m1, dd_m2, dd_m3, dd_q1, dd_q2, dd_q3, dd_y1, dd_y2, dd_y3
+;
+
+dd_m1:integer   | dd_m2:integer | dd_m3:integer | dd_q1:integer | dd_q2:integer | dd_q3:integer | dd_y1:integer | dd_y2:integer | dd_y3:integer
+12              | 12            | 12            | 4             | 4             | 4             | 1             | 1             | 1                                     
+;
+
+evalDateDiffErrorOutOfIntegerRange#[skip:-8.12.99, reason:date_diff added in 8.13]
+ROW date1=to_datetime("2023-12-02T11:00:00.000Z"), date2=to_datetime("2023-12-23T11:00:00.000Z") 
+| EVAL dd_oo=date_diff("nanoseconds", date1, date2)
+| keep dd_oo
+;
+warning: Line 2:14: evaluation of [date_diff(\"nanoseconds\", date1, date2)] failed, treating result as null. Only first 20 failures recorded.
+warning: Line 2:14: org.elasticsearch.xpack.ql.InvalidArgumentException: [1814400000000000] out of [integer] range
+
+dd_oo:integer
+null
+;
+
 evalDateParseWithSimpleDate
 row a = "2023-02-01" | eval b = date_parse("yyyy-MM-dd", a) | keep b;
 

x-pack/plugin/esql/qa/testFixtures/src/main/resources/date.csv-spec

@@ -26,6 +26,7 @@ cos                      |"double cos(n:integer|long|double|unsigned_long)"
 cosh                     |"double cosh(n:integer|long|double|unsigned_long)"     |n                        |"integer|long|double|unsigned_long"                 | "The number who's hyperbolic cosine is to be returned" |double                    | "Returns the hyperbolic cosine of a number"                      | false                | false
 count                    |? count(arg1:?)                                         |arg1                     |?                 | ""                                                 |?                    | ""                      | false                | false
 count_distinct           |? count_distinct(arg1:?, arg2:?)                        |[arg1, arg2]             |[?, ?]            |["", ""]                                            |?                    | ""                      | [false, false]       | false
+date_diff                |"integer date_diff(unit:keyword|text, startTimestamp:date, endTimestamp:date)"|[unit, startTimestamp, endTimestamp] |["keyword|text", "date", "date"]  |["A valid date unit", "A string representing a start timestamp", "A string representing an end timestamp"] |integer | "Subtract 2 dates and return their difference in multiples of a unit specified in the 1st argument" | [false, false, false] | false
 date_extract             |? date_extract(arg1:?, arg2:?)                          |[arg1, arg2]             |[?, ?]            |["", ""]                                            |?                    | ""                      | [false, false]       | false
 date_format              |? date_format(arg1:?, arg2:?)                           |[arg1, arg2]             |[?, ?]            |["", ""]                                            |?                    | ""                      | [false, false]       | false
 date_parse               |"date date_parse(?datePattern:keyword, dateString:keyword|text)"|[datePattern, dateString]|["keyword", "keyword|text"]|[A valid date pattern, A string representing a date]|date                 |Parses a string into a date value | [true, false]       | false         
@@ -117,6 +118,7 @@ synopsis:keyword
 "double cosh(n:integer|long|double|unsigned_long)"
 ? count(arg1:?)
 ? count_distinct(arg1:?, arg2:?)
+"integer date_diff(unit:keyword|text, startTimestamp:date, endTimestamp:date)"
 ? date_extract(arg1:?, arg2:?)
 ? date_format(arg1:?, arg2:?)
 "date date_parse(?datePattern:keyword, dateString:keyword|text)"
@@ -205,9 +207,9 @@ is_nan                   |boolean is_nan(n:double)
 
 
 // see https://github.com/elastic/elasticsearch/issues/102120
-countFunctions#[skip:-8.11.99]
+countFunctions#[skip:-8.12.99]
 show functions |  stats  a = count(*), b = count(*), c = count(*) |  mv_expand c;
 
 a:long | b:long | c:long
-84     | 84     | 84
+85     | 85     | 85
 ;

x-pack/plugin/esql/qa/testFixtures/src/main/resources/show.csv-spec

@@ -0,0 +1,154 @@
+// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+// or more contributor license agreements. Licensed under the Elastic License
+// 2.0; you may not use this file except in compliance with the Elastic License
+// 2.0.
+package org.elasticsearch.xpack.esql.expression.function.scalar.date;
+
+import java.lang.IllegalArgumentException;
+import java.lang.Override;
+import java.lang.String;
+import org.elasticsearch.compute.data.Block;
+import org.elasticsearch.compute.data.IntBlock;
+import org.elasticsearch.compute.data.LongBlock;
+import org.elasticsearch.compute.data.LongVector;
+import org.elasticsearch.compute.data.Page;
+import org.elasticsearch.compute.operator.DriverContext;
+import org.elasticsearch.compute.operator.EvalOperator;
+import org.elasticsearch.core.Releasables;
+import org.elasticsearch.xpack.esql.expression.function.Warnings;
+import org.elasticsearch.xpack.ql.InvalidArgumentException;
+import org.elasticsearch.xpack.ql.tree.Source;
+
+/**
+ * {@link EvalOperator.ExpressionEvaluator} implementation for {@link DateDiff}.
+ * This class is generated. Do not edit it.
+ */
+public final class DateDiffConstantEvaluator implements EvalOperator.ExpressionEvaluator {
+  private final Warnings warnings;
+
+  private final DateDiff.Part datePartFieldUnit;
+
+  private final EvalOperator.ExpressionEvaluator startTimestamp;
+
+  private final EvalOperator.ExpressionEvaluator endTimestamp;
+
+  private final DriverContext driverContext;
+
+  public DateDiffConstantEvaluator(Source source, DateDiff.Part datePartFieldUnit,
+      EvalOperator.ExpressionEvaluator startTimestamp,
+      EvalOperator.ExpressionEvaluator endTimestamp, DriverContext driverContext) {
+    this.warnings = new Warnings(source);
+    this.datePartFieldUnit = datePartFieldUnit;
+    this.startTimestamp = startTimestamp;
+    this.endTimestamp = endTimestamp;
+    this.driverContext = driverContext;
+  }
+
+  @Override
+  public Block eval(Page page) {
+    try (LongBlock startTimestampBlock = (LongBlock) startTimestamp.eval(page)) {
+      try (LongBlock endTimestampBlock = (LongBlock) endTimestamp.eval(page)) {
+        LongVector startTimestampVector = startTimestampBlock.asVector();
+        if (startTimestampVector == null) {
+          return eval(page.getPositionCount(), startTimestampBlock, endTimestampBlock);
+        }
+        LongVector endTimestampVector = endTimestampBlock.asVector();
+        if (endTimestampVector == null) {
+          return eval(page.getPositionCount(), startTimestampBlock, endTimestampBlock);
+        }
+        return eval(page.getPositionCount(), startTimestampVector, endTimestampVector);
+      }
+    }
+  }
+
+  public IntBlock eval(int positionCount, LongBlock startTimestampBlock,
+      LongBlock endTimestampBlock) {
+    try(IntBlock.Builder result = driverContext.blockFactory().newIntBlockBuilder(positionCount)) {
+      position: for (int p = 0; p < positionCount; p++) {
+        if (startTimestampBlock.isNull(p)) {
+          result.appendNull();
+          continue position;
+        }
+        if (startTimestampBlock.getValueCount(p) != 1) {
+          if (startTimestampBlock.getValueCount(p) > 1) {
+            warnings.registerException(new IllegalArgumentException("single-value function encountered multi-value"));
+          }
+          result.appendNull();
+          continue position;
+        }
+        if (endTimestampBlock.isNull(p)) {
+          result.appendNull();
+          continue position;
+        }
+        if (endTimestampBlock.getValueCount(p) != 1) {
+          if (endTimestampBlock.getValueCount(p) > 1) {
+            warnings.registerException(new IllegalArgumentException("single-value function encountered multi-value"));
+          }
+          result.appendNull();
+          continue position;
+        }
+        try {
+          result.appendInt(DateDiff.process(datePartFieldUnit, startTimestampBlock.getLong(startTimestampBlock.getFirstValueIndex(p)), endTimestampBlock.getLong(endTimestampBlock.getFirstValueIndex(p))));
+        } catch (IllegalArgumentException | InvalidArgumentException e) {
+          warnings.registerException(e);
+          result.appendNull();
+        }
+      }
+      return result.build();
+    }
+  }
+
+  public IntBlock eval(int positionCount, LongVector startTimestampVector,
+      LongVector endTimestampVector) {
+    try(IntBlock.Builder result = driverContext.blockFactory().newIntBlockBuilder(positionCount)) {
+      position: for (int p = 0; p < positionCount; p++) {
+        try {
+          result.appendInt(DateDiff.process(datePartFieldUnit, startTimestampVector.getLong(p), endTimestampVector.getLong(p)));
+        } catch (IllegalArgumentException | InvalidArgumentException e) {
+          warnings.registerException(e);
+          result.appendNull();
+        }
+      }
+      return result.build();
+    }
+  }
+
+  @Override
+  public String toString() {
+    return "DateDiffConstantEvaluator[" + "datePartFieldUnit=" + datePartFieldUnit + ", startTimestamp=" + startTimestamp + ", endTimestamp=" + endTimestamp + "]";
+  }
+
+  @Override
+  public void close() {
+    Releasables.closeExpectNoException(startTimestamp, endTimestamp);
+  }
+
+  static class Factory implements EvalOperator.ExpressionEvaluator.Factory {
+    private final Source source;
+
+    private final DateDiff.Part datePartFieldUnit;
+
+    private final EvalOperator.ExpressionEvaluator.Factory startTimestamp;
+
+    private final EvalOperator.ExpressionEvaluator.Factory endTimestamp;
+
+    public Factory(Source source, DateDiff.Part datePartFieldUnit,
+        EvalOperator.ExpressionEvaluator.Factory startTimestamp,
+        EvalOperator.ExpressionEvaluator.Factory endTimestamp) {
+      this.source = source;
+      this.datePartFieldUnit = datePartFieldUnit;
+      this.startTimestamp = startTimestamp;
+      this.endTimestamp = endTimestamp;
+    }
+
+    @Override
+    public DateDiffConstantEvaluator get(DriverContext context) {
+      return new DateDiffConstantEvaluator(source, datePartFieldUnit, startTimestamp.get(context), endTimestamp.get(context), context);
+    }
+
+    @Override
+    public String toString() {
+      return "DateDiffConstantEvaluator[" + "datePartFieldUnit=" + datePartFieldUnit + ", startTimestamp=" + startTimestamp + ", endTimestamp=" + endTimestamp + "]";
+    }
+  }
+}