ESQL: Align year diffing to the rest of the units in DATE_DIFF: chronological (#113103)

This will correct/switch "year" unit diffing from the current integer
subtraction to a crono subtraction. Consequently, two dates are (at
least) one year apart now if (at least) a full calendar year separates
them. The previous implementation simply subtracted the year part of the
dates.

Note: this parts with ES SQL's implementation of the same function,
which itself is aligned with MS SQL's implementation, which works
equivalent to an integer subtraction.

Fixes #112482.

Author: bpintea

docs/changelog/113103.yaml

@@ -0,0 +1,6 @@
+pr: 113103
+summary: "ESQL: Align year diffing to the rest of the units in DATE_DIFF: chronological"
+area: ES|QL
+type: bug
+issues:
+ - 112482

docs/reference/esql/functions/examples/date_diff.asciidoc

@@ -367,6 +367,80 @@ date1:date               | dd_ms:integer
 2023-12-02T11:00:00.000Z | 1000
 ;
 
+evalDateDiffMonthAsWhole0Months
+
+ROW from="2023-12-31T23:59:59.999Z"::DATETIME, to="2024-01-01T00:00:00"::DATETIME
+| EVAL msecs=DATE_DIFF("milliseconds", from, to), months=DATE_DIFF("month", from, to)
+;
+
+          from:date     |           to:date      |     msecs:integer|    months:integer
+2023-12-31T23:59:59.999Z|2024-01-01T00:00:00.000Z|1                 |0
+
+;
+
+evalDateDiffMonthAsWhole1Month
+
+ROW from="2023-12-31T23:59:59.999Z"::DATETIME, to="2024-02-01T00:00:00"::DATETIME
+| EVAL secs=DATE_DIFF("seconds", from, to), months=DATE_DIFF("month", from, to)
+;
+
+          from:date     |           to:date      |     secs:integer|    months:integer
+2023-12-31T23:59:59.999Z|2024-02-01T00:00:00.000Z|2678400          |1
+
+;
+
+evalDateDiffYearAsWhole0Years
+required_capability: date_diff_year_calendarial
+
+ROW from="2023-12-31T23:59:59.999Z"::DATETIME, to="2024-01-01T00:00:00"::DATETIME
+| EVAL msecs=DATE_DIFF("milliseconds", from, to), years=DATE_DIFF("year", from, to)
+;
+
+          from:date     |           to:date      |     msecs:integer |     years:integer
+2023-12-31T23:59:59.999Z|2024-01-01T00:00:00.000Z|1                  |0
+;
+
+evalDateDiffYearAsWhole1Year
+required_capability: date_diff_year_calendarial
+
+ROW from="2023-12-31T23:59:59.999Z"::DATETIME, to="2025-01-01T00:00:00"::DATETIME
+| EVAL secs=DATE_DIFF("seconds", from, to), years=DATE_DIFF("year", from, to)
+;
+
+          from:date     |           to:date      |     secs:integer|     years:integer
+2023-12-31T23:59:59.999Z|2025-01-01T00:00:00.000Z|31622400         |1
+;
+
+evalDateDiffYearAsWhole1Year
+required_capability: date_diff_year_calendarial
+
+ROW from="2024-01-01T00:00:00Z"::DATETIME, to="2025-01-01T00:00:00"::DATETIME
+| EVAL secs=DATE_DIFF("seconds", from, to), years=DATE_DIFF("year", from, to)
+;
+
+          from:date     |           to:date      |     secs:integer|     years:integer
+2024-01-01T00:00:00.000Z|2025-01-01T00:00:00.000Z|31622400         |1
+;
+
+evalDateDiffYearForDocs
+required_capability: date_diff_year_calendarial
+
+// tag::evalDateDiffYearForDocs[]
+ROW end_23="2023-12-31T23:59:59.999Z"::DATETIME,
+  start_24="2024-01-01T00:00:00.000Z"::DATETIME,
+    end_24="2024-12-31T23:59:59.999"::DATETIME
+| EVAL end23_to_start24=DATE_DIFF("year", end_23, start_24)
+| EVAL end23_to_end24=DATE_DIFF("year", end_23, end_24)
+| EVAL start_to_end_24=DATE_DIFF("year", start_24, end_24)
+// end::evalDateDiffYearForDocs[]
+;
+
+// tag::evalDateDiffYearForDocs-result[]
+         end_23:date    |        start_24:date   |         end_24:date    |end23_to_start24:integer|end23_to_end24:integer|start_to_end_24:integer
+2023-12-31T23:59:59.999Z|2024-01-01T00:00:00.000Z|2024-12-31T23:59:59.999Z|0                       |1                     |0
+// end::evalDateDiffYearForDocs-result[]
+;
+
 evalDateParseWithSimpleDate
 row a = "2023-02-01" | eval b = date_parse("yyyy-MM-dd", a) | keep b;
 

docs/reference/esql/functions/kibana/definition/date_diff.json

@@ -320,7 +320,12 @@ public enum Cap {
          * Don't optimize CASE IS NOT NULL function by not requiring the fields to be not null as well.
          * https://github.com/elastic/elasticsearch/issues/112704
          */
-        FIXED_WRONG_IS_NOT_NULL_CHECK_ON_CASE;
+        FIXED_WRONG_IS_NOT_NULL_CHECK_ON_CASE,
+
+        /**
+         * Compute year differences in full calendar years.
+         */
+        DATE_DIFF_YEAR_CALENDARIAL;
 
         private final boolean snapshotOnly;
         private final FeatureFlag featureFlag;

x-pack/plugin/esql/qa/testFixtures/src/main/resources/date.csv-spec

@@ -66,7 +66,7 @@ public class DateDiff extends EsqlScalarFunction {
      */
     public enum Part implements DateTimeField {
 
-        YEAR((start, end) -> end.getYear() - start.getYear(), "years", "yyyy", "yy"),
+        YEAR((start, end) -> safeToInt(ChronoUnit.YEARS.between(start, end)), "years", "yyyy", "yy"),
         QUARTER((start, end) -> safeToInt(IsoFields.QUARTER_YEARS.between(start, end)), "quarters", "qq", "q"),
         MONTH((start, end) -> safeToInt(ChronoUnit.MONTHS.between(start, end)), "months", "mm", "m"),
         DAYOFYEAR((start, end) -> safeToInt(ChronoUnit.DAYS.between(start, end)), "dy", "y"),
@@ -126,36 +126,44 @@ public static Part resolve(String dateTimeUnit) {
         }
     }
 
-    @FunctionInfo(returnType = "integer", description = """
-        Subtracts the `startTimestamp` from the `endTimestamp` and returns the difference in multiples of `unit`.
-        If `startTimestamp` is later than the `endTimestamp`, negative values are returned.""", detailedDescription = """
-        [cols=\"^,^\",role=\"styled\"]
-        |===
-        2+h|Datetime difference units
-
-        s|unit
-        s|abbreviations
-
-        | year        | years, yy, yyyy
-        | quarter     | quarters, qq, q
-        | month       | months, mm, m
-        | dayofyear   | dy, y
-        | day         | days, dd, d
-        | week        | weeks, wk, ww
-        | weekday     | weekdays, dw
-        | hour        | hours, hh
-        | minute      | minutes, mi, n
-        | second      | seconds, ss, s
-        | millisecond | milliseconds, ms
-        | microsecond | microseconds, mcs
-        | nanosecond  | nanoseconds, ns
-        |===
-
-        Note that while there is an overlap between the function's supported units and
-        {esql}'s supported time span literals, these sets are distinct and not
-        interchangeable. Similarly, the supported abbreviations are conveniently shared
-        with implementations of this function in other established products and not
-        necessarily common with the date-time nomenclature used by {es}.""", examples = @Example(file = "date", tag = "docsDateDiff"))
+    @FunctionInfo(
+        returnType = "integer",
+        description = """
+            Subtracts the `startTimestamp` from the `endTimestamp` and returns the difference in multiples of `unit`.
+            If `startTimestamp` is later than the `endTimestamp`, negative values are returned.""",
+        detailedDescription = """
+            [cols=\"^,^\",role=\"styled\"]
+            |===
+            2+h|Datetime difference units
+
+            s|unit
+            s|abbreviations
+
+            | year        | years, yy, yyyy
+            | quarter     | quarters, qq, q
+            | month       | months, mm, m
+            | dayofyear   | dy, y
+            | day         | days, dd, d
+            | week        | weeks, wk, ww
+            | weekday     | weekdays, dw
+            | hour        | hours, hh
+            | minute      | minutes, mi, n
+            | second      | seconds, ss, s
+            | millisecond | milliseconds, ms
+            | microsecond | microseconds, mcs
+            | nanosecond  | nanoseconds, ns
+            |===
+
+            Note that while there is an overlap between the function's supported units and
+            {esql}'s supported time span literals, these sets are distinct and not
+            interchangeable. Similarly, the supported abbreviations are conveniently shared
+            with implementations of this function in other established products and not
+            necessarily common with the date-time nomenclature used by {es}.""",
+        examples = { @Example(file = "date", tag = "docsDateDiff"), @Example(description = """
+            When subtracting in calendar units - like year, month a.s.o. - only the fully elapsed units are counted.
+            To avoid this and obtain also remainders, simply switch to the next smaller unit and do the date math accordingly.
+            """, file = "date", tag = "evalDateDiffYearForDocs") }
+    )
     public DateDiff(
         Source source,
         @Param(name = "unit", type = { "keyword", "text" }, description = "Time difference unit") Expression unit,

x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java

@@ -113,6 +113,34 @@ public static Iterable<Object[]> parameters() {
                 )
             )
         );
+        suppliers.add(new TestCaseSupplier("Date Diff In Year - 1", List.of(DataType.KEYWORD, DataType.DATETIME, DataType.DATETIME), () -> {
+            ZonedDateTime zdtStart2 = ZonedDateTime.parse("2023-12-12T00:01:01Z");
+            ZonedDateTime zdtEnd2 = ZonedDateTime.parse("2024-12-12T00:01:01Z");
+            return new TestCaseSupplier.TestCase(
+                List.of(
+                    new TestCaseSupplier.TypedData(new BytesRef("year"), DataType.KEYWORD, "unit"),
+                    new TestCaseSupplier.TypedData(zdtStart2.toInstant().toEpochMilli(), DataType.DATETIME, "startTimestamp"),
+                    new TestCaseSupplier.TypedData(zdtEnd2.toInstant().toEpochMilli(), DataType.DATETIME, "endTimestamp")
+                ),
+                "DateDiffEvaluator[unit=Attribute[channel=0], startTimestamp=Attribute[channel=1], " + "endTimestamp=Attribute[channel=2]]",
+                DataType.INTEGER,
+                equalTo(1)
+            );
+        }));
+        suppliers.add(new TestCaseSupplier("Date Diff In Year - 0", List.of(DataType.KEYWORD, DataType.DATETIME, DataType.DATETIME), () -> {
+            ZonedDateTime zdtStart2 = ZonedDateTime.parse("2023-12-12T00:01:01.001Z");
+            ZonedDateTime zdtEnd2 = ZonedDateTime.parse("2024-12-12T00:01:01Z");
+            return new TestCaseSupplier.TestCase(
+                List.of(
+                    new TestCaseSupplier.TypedData(new BytesRef("year"), DataType.KEYWORD, "unit"),
+                    new TestCaseSupplier.TypedData(zdtStart2.toInstant().toEpochMilli(), DataType.DATETIME, "startTimestamp"),
+                    new TestCaseSupplier.TypedData(zdtEnd2.toInstant().toEpochMilli(), DataType.DATETIME, "endTimestamp")
+                ),
+                "DateDiffEvaluator[unit=Attribute[channel=0], startTimestamp=Attribute[channel=1], " + "endTimestamp=Attribute[channel=2]]",
+                DataType.INTEGER,
+                equalTo(0)
+            );
+        }));
         return parameterSuppliersFromTypedData(anyNullIsNull(false, suppliers));
     }