diff --git a/docs/changelog/127962.yaml b/docs/changelog/127962.yaml new file mode 100644 index 0000000000000..f96b79a69d5b3 --- /dev/null +++ b/docs/changelog/127962.yaml @@ -0,0 +1,6 @@ +pr: 127962 +summary: Support DATE_NANOS in LOOKUP JOIN +area: ES|QL +type: bug +issues: + - 127249 diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java index ec12a183fa5c1..636683d2b5778 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java @@ -472,19 +472,12 @@ public DateFieldType(String name) { this(name, true, true, false, true, DEFAULT_DATE_TIME_FORMATTER, Resolution.MILLISECONDS, null, null, Collections.emptyMap()); } + public DateFieldType(String name, boolean isIndexed, Resolution resolution) { + this(name, isIndexed, isIndexed, false, true, DEFAULT_DATE_TIME_FORMATTER, resolution, null, null, Collections.emptyMap()); + } + public DateFieldType(String name, boolean isIndexed) { - this( - name, - isIndexed, - isIndexed, - false, - true, - DEFAULT_DATE_TIME_FORMATTER, - Resolution.MILLISECONDS, - null, - null, - Collections.emptyMap() - ); + this(name, isIndexed, Resolution.MILLISECONDS); } public DateFieldType(String name, DateFormatter dateFormatter) { @@ -698,6 +691,54 @@ public static long parseToLong( return resolution.convert(dateParser.parse(BytesRefs.toString(value), now, roundUp, zone)); } + /** + * Similar to the {@link DateFieldType#termQuery} method, but works on dates that are already parsed to a long + * in the same precision as the field mapper. + */ + public Query equalityQuery(Long value, @Nullable SearchExecutionContext context) { + return rangeQuery(value, value, true, true, context); + } + + /** + * Similar to the existing + * {@link DateFieldType#rangeQuery(Object, Object, boolean, boolean, ShapeRelation, ZoneId, DateMathParser, SearchExecutionContext)} + * method, but works on dates that are already parsed to a long in the same precision as the field mapper. + */ + public Query rangeQuery( + Long lowerTerm, + Long upperTerm, + boolean includeLower, + boolean includeUpper, + SearchExecutionContext context + ) { + failIfNotIndexedNorDocValuesFallback(context); + long l, u; + if (lowerTerm == null) { + l = Long.MIN_VALUE; + } else { + l = (includeLower == false) ? lowerTerm + 1 : lowerTerm; + } + if (upperTerm == null) { + u = Long.MAX_VALUE; + } else { + u = (includeUpper == false) ? upperTerm - 1 : upperTerm; + } + Query query; + if (isIndexed()) { + query = LongPoint.newRangeQuery(name(), l, u); + if (hasDocValues()) { + Query dvQuery = SortedNumericDocValuesField.newSlowRangeQuery(name(), l, u); + query = new IndexOrDocValuesQuery(query, dvQuery); + } + } else { + query = SortedNumericDocValuesField.newSlowRangeQuery(name(), l, u); + } + if (hasDocValues() && context.indexSortedOnField(name())) { + query = new XIndexSortSortedNumericDocValuesRangeQuery(name(), l, u, query); + } + return query; + } + @Override public Query distanceFeatureQuery(Object origin, String pivot, SearchExecutionContext context) { failIfNotIndexedNorDocValuesFallback(context); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/DateFieldTypeTests.java b/server/src/test/java/org/elasticsearch/index/mapper/DateFieldTypeTests.java index d12a0ec12d733..174f104100399 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/DateFieldTypeTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/DateFieldTypeTests.java @@ -208,13 +208,11 @@ public void testValueForSearch() { assertEquals(date, ft.valueForDisplay(instant)); } + /** + * If the term field is a string of date-time format with exact seconds (no sub-seconds), any data within a 1second range will match. + */ public void testTermQuery() { - Settings indexSettings = indexSettings(IndexVersion.current(), 1, 1).build(); - SearchExecutionContext context = SearchExecutionContextHelper.createSimple( - new IndexSettings(IndexMetadata.builder("foo").settings(indexSettings).build(), indexSettings), - parserConfig(), - writableRegistry() - ); + SearchExecutionContext context = prepareIndexForTermQuery(); MappedFieldType ft = new DateFieldType("field"); String date = "2015-10-12T14:10:55"; long instant = DateFormatters.from(DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.parse(date)).toInstant().toEpochMilli(); @@ -228,45 +226,99 @@ public void testTermQuery() { expected = SortedNumericDocValuesField.newSlowRangeQuery("field", instant, instant + 999); assertEquals(expected, ft.termQuery(date, context)); - MappedFieldType unsearchable = new DateFieldType( - "field", - false, - false, - false, - DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER, - Resolution.MILLISECONDS, - null, - null, - Collections.emptyMap() + assertIndexUnsearchable(Resolution.MILLISECONDS, (unsearchable) -> unsearchable.termQuery(date, context)); + } + + /** + * If the term field is a string of date-time format with sub-seconds, only data with exact ms precision will match. + */ + public void testTermQuerySubseconds() { + SearchExecutionContext context = prepareIndexForTermQuery(); + MappedFieldType ft = new DateFieldType("field"); + String date = "2015-10-12T14:10:55.01"; + long instant = DateFormatters.from(DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.parse(date)).toInstant().toEpochMilli(); + Query expected = new IndexOrDocValuesQuery( + LongPoint.newRangeQuery("field", instant, instant), + SortedNumericDocValuesField.newSlowRangeQuery("field", instant, instant) ); - IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> unsearchable.termQuery(date, context)); - assertEquals("Cannot search on field [field] since it is not indexed nor has doc values.", e.getMessage()); + assertEquals(expected, ft.termQuery(date, context)); + + ft = new DateFieldType("field", false); + expected = SortedNumericDocValuesField.newSlowRangeQuery("field", instant, instant); + assertEquals(expected, ft.termQuery(date, context)); + + assertIndexUnsearchable(Resolution.MILLISECONDS, (unsearchable) -> unsearchable.termQuery(date, context)); } - public void testRangeQuery() throws IOException { - Settings indexSettings = indexSettings(IndexVersion.current(), 1, 1).build(); - SearchExecutionContext context = new SearchExecutionContext( - 0, - 0, - new IndexSettings(IndexMetadata.builder("foo").settings(indexSettings).build(), indexSettings), - null, - null, - null, - MappingLookup.EMPTY, - null, - null, - parserConfig(), - writableRegistry(), - null, - null, - () -> nowInMillis, - null, - null, - () -> true, - null, - Collections.emptyMap(), - MapperMetrics.NOOP + /** + * If the term field is a string of the long value (ms since epoch), only data with exact ms precision will match. + */ + public void testTermQueryMillis() { + SearchExecutionContext context = prepareIndexForTermQuery(); + MappedFieldType ft = new DateFieldType("field"); + String date = "2015-10-12T14:10:55"; + long instant = DateFormatters.from(DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.parse(date)).toInstant().toEpochMilli(); + Query expected = new IndexOrDocValuesQuery( + LongPoint.newRangeQuery("field", instant, instant), + SortedNumericDocValuesField.newSlowRangeQuery("field", instant, instant) ); + assertEquals(expected, ft.termQuery(instant, context)); + + ft = new DateFieldType("field", false); + expected = SortedNumericDocValuesField.newSlowRangeQuery("field", instant, instant); + assertEquals(expected, ft.termQuery(instant, context)); + + assertIndexUnsearchable(Resolution.MILLISECONDS, (unsearchable) -> unsearchable.termQuery(instant, context)); + } + + /** + * This query has similar behaviour to passing a String containing a long to termQuery, only data with exact ms precision will match. + */ + public void testEqualityQuery() { + SearchExecutionContext context = prepareIndexForTermQuery(); + DateFieldType ft = new DateFieldType("field"); + String date = "2015-10-12T14:10:55"; + long instant = DateFormatters.from(DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.parse(date)).toInstant().toEpochMilli(); + Query expected = new IndexOrDocValuesQuery( + LongPoint.newRangeQuery("field", instant, instant), + SortedNumericDocValuesField.newSlowRangeQuery("field", instant, instant) + ); + assertEquals(expected, ft.equalityQuery(instant, context)); + + ft = new DateFieldType("field", false); + expected = SortedNumericDocValuesField.newSlowRangeQuery("field", instant, instant); + assertEquals(expected, ft.equalityQuery(instant, context)); + + assertIndexUnsearchable(Resolution.MILLISECONDS, (unsearchable) -> unsearchable.equalityQuery(instant, context)); + } + + /** + * This query supports passing a ns value, and only data with exact ns precision will match. + */ + public void testEqualityNanosQuery() { + SearchExecutionContext context = prepareIndexForTermQuery(); + DateFieldType ft = new DateFieldType("field", Resolution.NANOSECONDS); + String date = "2015-10-12T14:10:55"; + long instant = DateFormatters.from(DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.parse(date)).toInstant().toEpochMilli() * 1000000L; + Query expected = new IndexOrDocValuesQuery( + LongPoint.newRangeQuery("field", instant, instant), + SortedNumericDocValuesField.newSlowRangeQuery("field", instant, instant) + ); + assertEquals(expected, ft.equalityQuery(instant, context)); + + ft = new DateFieldType("field", false); + expected = SortedNumericDocValuesField.newSlowRangeQuery("field", instant, instant); + assertEquals(expected, ft.equalityQuery(instant, context)); + + assertIndexUnsearchable(Resolution.NANOSECONDS, (unsearchable) -> unsearchable.equalityQuery(instant, context)); + } + + /** + * If the term fields are strings of date-time format with exact seconds (no sub-seconds), + * the second field will be rounded up to the next second. + */ + public void testRangeQuery() throws IOException { + SearchExecutionContext context = prepareIndexForRangeQuery(); MappedFieldType ft = new DateFieldType("field"); String date1 = "2015-10-12T14:10:55"; String date2 = "2016-04-28T11:33:52"; @@ -298,22 +350,105 @@ public void testRangeQuery() throws IOException { expected2 = new DateRangeIncludingNowQuery(SortedNumericDocValuesField.newSlowRangeQuery("field", instant1, instant2)); assertEquals(expected2, ft2.rangeQuery("now", instant2, true, true, null, null, null, context)); - MappedFieldType unsearchable = new DateFieldType( - "field", - false, - false, - false, - DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER, + assertIndexUnsearchable( Resolution.MILLISECONDS, - null, - null, - Collections.emptyMap() + (unsearchable) -> unsearchable.rangeQuery(date1, date2, true, true, null, null, null, context) ); - IllegalArgumentException e = expectThrows( - IllegalArgumentException.class, - () -> unsearchable.rangeQuery(date1, date2, true, true, null, null, null, context) + } + + /** + * If the term fields are strings of date-time format with sub-seconds, + * the lower and upper values will be matched inclusively to the ms. + */ + public void testRangeQuerySubseconds() throws IOException { + SearchExecutionContext context = prepareIndexForRangeQuery(); + MappedFieldType ft = new DateFieldType("field"); + String date1 = "2015-10-12T14:10:55.01"; + String date2 = "2016-04-28T11:33:52.01"; + long instant1 = DateFormatters.from(DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.parse(date1)).toInstant().toEpochMilli(); + long instant2 = DateFormatters.from(DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.parse(date2)).toInstant().toEpochMilli(); + Query expected = new IndexOrDocValuesQuery( + LongPoint.newRangeQuery("field", instant1, instant2), + SortedNumericDocValuesField.newSlowRangeQuery("field", instant1, instant2) + ); + assertEquals(expected, ft.rangeQuery(date1, date2, true, true, null, null, null, context).rewrite(newSearcher(new MultiReader()))); + + MappedFieldType ft2 = new DateFieldType("field", false); + Query expected2 = SortedNumericDocValuesField.newSlowRangeQuery("field", instant1, instant2); + assertEquals( + expected2, + ft2.rangeQuery(date1, date2, true, true, null, null, null, context).rewrite(newSearcher(new MultiReader())) + ); + + instant1 = nowInMillis; + instant2 = instant1 + 100; + expected = new DateRangeIncludingNowQuery( + new IndexOrDocValuesQuery( + LongPoint.newRangeQuery("field", instant1, instant2), + SortedNumericDocValuesField.newSlowRangeQuery("field", instant1, instant2) + ) + ); + assertEquals(expected, ft.rangeQuery("now", instant2, true, true, null, null, null, context)); + + expected2 = new DateRangeIncludingNowQuery(SortedNumericDocValuesField.newSlowRangeQuery("field", instant1, instant2)); + assertEquals(expected2, ft2.rangeQuery("now", instant2, true, true, null, null, null, context)); + + assertIndexUnsearchable( + Resolution.MILLISECONDS, + (unsearchable) -> unsearchable.rangeQuery(date1, date2, true, true, null, null, null, context) ); - assertEquals("Cannot search on field [field] since it is not indexed nor has doc values.", e.getMessage()); + } + + /** + * If the term fields are strings of long ms, the lower and upper values will be matched inclusively to the ms. + */ + public void testRangeQueryMillis() throws IOException { + SearchExecutionContext context = prepareIndexForRangeQuery(); + DateFieldType ft = new DateFieldType("field"); + String date1 = "2015-10-12T14:10:55.01"; + String date2 = "2016-04-28T11:33:52.01"; + long instant1 = DateFormatters.from(DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.parse(date1)).toInstant().toEpochMilli(); + long instant2 = DateFormatters.from(DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.parse(date2)).toInstant().toEpochMilli(); + Query expected = new IndexOrDocValuesQuery( + LongPoint.newRangeQuery("field", instant1, instant2), + SortedNumericDocValuesField.newSlowRangeQuery("field", instant1, instant2) + ); + assertEquals(expected, ft.rangeQuery(instant1, instant2, true, true, context).rewrite(newSearcher(new MultiReader()))); + + DateFieldType ft2 = new DateFieldType("field", false); + Query expected2 = SortedNumericDocValuesField.newSlowRangeQuery("field", instant1, instant2); + assertEquals(expected2, ft2.rangeQuery(instant1, instant2, true, true, context).rewrite(newSearcher(new MultiReader()))); + + assertIndexUnsearchable( + Resolution.MILLISECONDS, + (unsearchable) -> unsearchable.rangeQuery(instant1, instant2, true, true, context) + ); + } + + /** + * If the term fields are strings of long ns, the lower and upper values will be matched inclusively to the ns. + */ + public void testRangeQueryNanos() throws IOException { + SearchExecutionContext context = prepareIndexForRangeQuery(); + DateFieldType ft = new DateFieldType("field", Resolution.NANOSECONDS); + String date1 = "2015-10-12T14:10:55.01"; + String date2 = "2016-04-28T11:33:52.01"; + long instant1 = DateFormatters.from(DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.parse(date1)).toInstant().toEpochMilli() * 1000000L; + long instant2 = DateFormatters.from(DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.parse(date2)).toInstant().toEpochMilli() * 1000000L; + Query expected = new IndexOrDocValuesQuery( + LongPoint.newRangeQuery("field", instant1, instant2), + SortedNumericDocValuesField.newSlowRangeQuery("field", instant1, instant2) + ); + assertEquals(expected, ft.rangeQuery(instant1, instant2, true, true, context).rewrite(newSearcher(new MultiReader()))); + + DateFieldType ft2 = new DateFieldType("field", false, Resolution.NANOSECONDS); + Query expected2 = SortedNumericDocValuesField.newSlowRangeQuery("field", instant1, instant2); + assertEquals( + expected2, + ft2.rangeQuery(date1, date2, true, true, null, null, null, context).rewrite(newSearcher(new MultiReader())) + ); + + assertIndexUnsearchable(Resolution.NANOSECONDS, (unsearchable) -> unsearchable.rangeQuery(instant1, instant2, true, true, context)); } public void testRangeQueryWithIndexSort() { @@ -420,4 +555,55 @@ public void testParseSourceValueNanos() throws IOException { MappedFieldType nullValueMapper = fieldType(Resolution.NANOSECONDS, "strict_date_time||epoch_millis", nullValueDate); assertEquals(List.of(nullValueDate), fetchSourceValue(nullValueMapper, null)); } + + private SearchExecutionContext prepareIndexForTermQuery() { + Settings indexSettings = indexSettings(IndexVersion.current(), 1, 1).build(); + return SearchExecutionContextHelper.createSimple( + new IndexSettings(IndexMetadata.builder("foo").settings(indexSettings).build(), indexSettings), + parserConfig(), + writableRegistry() + ); + } + + private SearchExecutionContext prepareIndexForRangeQuery() { + Settings indexSettings = indexSettings(IndexVersion.current(), 1, 1).build(); + return new SearchExecutionContext( + 0, + 0, + new IndexSettings(IndexMetadata.builder("foo").settings(indexSettings).build(), indexSettings), + null, + null, + null, + MappingLookup.EMPTY, + null, + null, + parserConfig(), + writableRegistry(), + null, + null, + () -> nowInMillis, + null, + null, + () -> true, + null, + Collections.emptyMap(), + MapperMetrics.NOOP + ); + } + + private void assertIndexUnsearchable(Resolution resolution, ThrowingConsumer runnable) { + DateFieldType unsearchable = new DateFieldType( + "field", + false, + false, + false, + DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER, + resolution, + null, + null, + Collections.emptyMap() + ); + IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> runnable.accept(unsearchable)); + assertEquals("Cannot search on field [field] since it is not indexed nor has doc values.", e.getMessage()); + } } diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/lookup/QueryList.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/lookup/QueryList.java index 5d359e2fb612f..d6fb6e4bfcdec 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/lookup/QueryList.java +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/lookup/QueryList.java @@ -11,6 +11,7 @@ import org.apache.lucene.geo.GeoEncodingUtils; import org.apache.lucene.search.BooleanClause; import org.apache.lucene.search.BooleanQuery; +import org.apache.lucene.search.ConstantScoreQuery; import org.apache.lucene.search.MatchAllDocsQuery; import org.apache.lucene.search.Query; import org.apache.lucene.util.BytesRef; @@ -30,6 +31,7 @@ import org.elasticsearch.geometry.Point; import org.elasticsearch.geometry.utils.GeometryValidator; import org.elasticsearch.geometry.utils.WellKnownBinary; +import org.elasticsearch.index.mapper.DateFieldMapper; import org.elasticsearch.index.mapper.GeoShapeQueryable; import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.RangeFieldMapper; @@ -37,7 +39,9 @@ import java.io.IOException; import java.util.ArrayList; +import java.util.HashSet; import java.util.List; +import java.util.Set; import java.util.function.IntFunction; /** @@ -189,6 +193,14 @@ public static QueryList dateTermQueryList(MappedFieldType field, SearchExecution ); } + /** + * Returns a list of term queries for the given field and the input block of + * {@code date_nanos} field values. + */ + public static QueryList dateNanosTermQueryList(MappedFieldType field, SearchExecutionContext searchExecutionContext, LongBlock block) { + return new DateNanosQueryList(field, searchExecutionContext, block, false); + } + /** * Returns a list of geo_shape queries for the given field and the input block. */ @@ -232,6 +244,68 @@ Query doGetQuery(int position, int firstValueIndex, int valueCount) { } } + private static class DateNanosQueryList extends QueryList { + protected final IntFunction blockValueReader; + private final DateFieldMapper.DateFieldType dateFieldType; + + private DateNanosQueryList( + MappedFieldType field, + SearchExecutionContext searchExecutionContext, + LongBlock block, + boolean onlySingleValues + ) { + super(field, searchExecutionContext, block, onlySingleValues); + if (field instanceof RangeFieldMapper.RangeFieldType rangeFieldType) { + // TODO: do this validation earlier + throw new IllegalArgumentException( + "DateNanosQueryList does not support range fields [" + rangeFieldType + "]: " + field.name() + ); + } + this.blockValueReader = block::getLong; + if (field instanceof DateFieldMapper.DateFieldType dateFieldType) { + // Validate that the field is a date_nanos field + // TODO: Consider allowing date_nanos to match normal datetime fields + if (dateFieldType.resolution() != DateFieldMapper.Resolution.NANOSECONDS) { + throw new IllegalArgumentException( + "DateNanosQueryList only supports date_nanos fields, but got: " + field.typeName() + " for field: " + field.name() + ); + } + this.dateFieldType = dateFieldType; + } else { + throw new IllegalArgumentException( + "DateNanosQueryList only supports date_nanos fields, but got: " + field.typeName() + " for field: " + field.name() + ); + } + } + + @Override + public DateNanosQueryList onlySingleValues() { + return new DateNanosQueryList(field, searchExecutionContext, (LongBlock) block, true); + } + + @Override + Query doGetQuery(int position, int firstValueIndex, int valueCount) { + return switch (valueCount) { + case 0 -> null; + case 1 -> dateFieldType.equalityQuery(blockValueReader.apply(firstValueIndex), searchExecutionContext); + default -> { + // The following code is a slight simplification of the DateFieldMapper.termsQuery method + final Set values = new HashSet<>(valueCount); + BooleanQuery.Builder builder = new BooleanQuery.Builder(); + for (int i = 0; i < valueCount; i++) { + final Long value = blockValueReader.apply(firstValueIndex + i); + if (values.contains(value)) { + continue; // Skip duplicates + } + values.add(value); + builder.add(dateFieldType.equalityQuery(value, searchExecutionContext), BooleanClause.Occur.SHOULD); + } + yield new ConstantScoreQuery(builder.build()); + } + }; + } + } + private static class GeoShapeQueryList extends QueryList { private final BytesRef scratch = new BytesRef(); private final IntFunction blockValueReader; diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/CsvAssert.java b/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/CsvAssert.java index 692c385cef216..1d669edfeea35 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/CsvAssert.java +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/CsvAssert.java @@ -278,7 +278,7 @@ private static void dataFailure( fail(description + System.lineSeparator() + describeFailures(dataFailures) + actual + expected); } - private static final int MAX_ROWS = 25; + private static final int MAX_ROWS = 50; private static String pipeTable( String description, diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/CsvTestsDataLoader.java b/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/CsvTestsDataLoader.java index 6e1cdaa90b905..34288b12ce5b3 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/CsvTestsDataLoader.java +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/CsvTestsDataLoader.java @@ -83,6 +83,8 @@ public class CsvTestsDataLoader { private static final TestDataset SAMPLE_DATA_TS_NANOS = SAMPLE_DATA.withIndex("sample_data_ts_nanos") .withData("sample_data_ts_nanos.csv") .withTypeMapping(Map.of("@timestamp", "date_nanos")); + private static final TestDataset LOOKUP_SAMPLE_DATA_TS_NANOS = SAMPLE_DATA_TS_NANOS.withIndex("lookup_sample_data_ts_nanos") + .withSetting("lookup-settings.json"); private static final TestDataset MISSING_IP_SAMPLE_DATA = new TestDataset("missing_ip_sample_data"); private static final TestDataset SAMPLE_DATA_PARTIAL_MAPPING = new TestDataset("partial_mapping_sample_data"); private static final TestDataset SAMPLE_DATA_NO_MAPPING = new TestDataset( @@ -160,6 +162,7 @@ public class CsvTestsDataLoader { Map.entry(SAMPLE_DATA_STR.indexName, SAMPLE_DATA_STR), Map.entry(SAMPLE_DATA_TS_LONG.indexName, SAMPLE_DATA_TS_LONG), Map.entry(SAMPLE_DATA_TS_NANOS.indexName, SAMPLE_DATA_TS_NANOS), + Map.entry(LOOKUP_SAMPLE_DATA_TS_NANOS.indexName, LOOKUP_SAMPLE_DATA_TS_NANOS), Map.entry(MISSING_IP_SAMPLE_DATA.indexName, MISSING_IP_SAMPLE_DATA), Map.entry(CLIENT_IPS.indexName, CLIENT_IPS), Map.entry(CLIENT_IPS_LOOKUP.indexName, CLIENT_IPS_LOOKUP), diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/lookup-join.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/lookup-join.csv-spec index 63a47424ac314..7cad510ec97db 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/lookup-join.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/lookup-join.csv-spec @@ -1661,3 +1661,27 @@ max:long 3450233 8268153 ; + +############################################### +# LOOKUP JOIN on date_nanos field +############################################### + +joinDateNanos +required_capability: join_lookup_v12 +required_capability: date_nanos_lookup_join + +FROM sample_data_ts_nanos +| LOOKUP JOIN lookup_sample_data_ts_nanos ON @timestamp +| KEEP @timestamp, client_ip, event_duration, message +| SORT @timestamp DESC +; + +@timestamp:date_nanos | client_ip:ip | event_duration:long | message:keyword +2023-10-23T13:55:01.543123456Z | 172.21.3.15 | 1756467 | Connected to 10.1.0.1 +2023-10-23T13:53:55.832123456Z | 172.21.3.15 | 5033755 | Connection error +2023-10-23T13:52:55.015123456Z | 172.21.3.15 | 8268153 | Connection error +2023-10-23T13:51:54.732123456Z | 172.21.3.15 | 725448 | Connection error +2023-10-23T13:33:34.937123456Z | 172.21.0.5 | 1232382 | Disconnected +2023-10-23T12:27:28.948123456Z | 172.21.2.113 | 2764889 | Connected to 10.1.0.2 +2023-10-23T12:15:03.360123456Z | 172.21.2.162 | 3450233 | Connected to 10.1.0.3 +; diff --git a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/LookupJoinTypesIT.java b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/LookupJoinTypesIT.java index ea31c37306d11..978069469c427 100644 --- a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/LookupJoinTypesIT.java +++ b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/LookupJoinTypesIT.java @@ -40,6 +40,7 @@ import static org.elasticsearch.xpack.esql.core.type.DataType.BOOLEAN; import static org.elasticsearch.xpack.esql.core.type.DataType.BYTE; import static org.elasticsearch.xpack.esql.core.type.DataType.DATETIME; +import static org.elasticsearch.xpack.esql.core.type.DataType.DATE_NANOS; import static org.elasticsearch.xpack.esql.core.type.DataType.DOC_DATA_TYPE; import static org.elasticsearch.xpack.esql.core.type.DataType.DOUBLE; import static org.elasticsearch.xpack.esql.core.type.DataType.FLOAT; @@ -175,6 +176,19 @@ protected Collection> nodePlugins() { } } + // Tests for mixed-date/time types + var dateTypes = List.of(DATETIME, DATE_NANOS); + { + TestConfigs configs = testConfigurations.computeIfAbsent("mixed-temporal", TestConfigs::new); + for (DataType mainType : dateTypes) { + for (DataType lookupType : dateTypes) { + if (mainType != lookupType) { + configs.addFails(mainType, lookupType); + } + } + } + } + // Tests for all unsupported types DataType[] unsupported = Join.UNSUPPORTED_TYPES; { @@ -201,7 +215,20 @@ protected Collection> nodePlugins() { } // Tests for all types where left and right are the same type - DataType[] supported = { BOOLEAN, LONG, INTEGER, DOUBLE, SHORT, BYTE, FLOAT, HALF_FLOAT, DATETIME, IP, KEYWORD, SCALED_FLOAT }; + DataType[] supported = { + BOOLEAN, + LONG, + INTEGER, + DOUBLE, + SHORT, + BYTE, + FLOAT, + HALF_FLOAT, + DATETIME, + DATE_NANOS, + IP, + KEYWORD, + SCALED_FLOAT }; { Collection existing = testConfigurations.values(); TestConfigs configs = testConfigurations.computeIfAbsent("same", TestConfigs::new); @@ -277,6 +304,10 @@ public void testLookupJoinMixedNumerical() { testLookupJoinTypes("mixed-numerical"); } + public void testLookupJoinMixedTemporal() { + testLookupJoinTypes("mixed-temporal"); + } + public void testLookupJoinSame() { testLookupJoinTypes("same"); } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java index 99aabaa34da9c..0c47e24e22d68 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java @@ -553,6 +553,12 @@ public enum Cap { * e.g. {@code WHERE millis > to_datenanos("2023-10-23T12:15:03.360103847") AND millis < to_datetime("2023-10-23T13:53:55.832")} */ FIX_DATE_NANOS_MIXED_RANGE_PUSHDOWN_BUG(), + + /** + * Support for date nanos in lookup join. Done in #127962 + */ + DATE_NANOS_LOOKUP_JOIN, + /** * DATE_PARSE supports reading timezones */ diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/enrich/AbstractLookupService.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/enrich/AbstractLookupService.java index c1a0ab7ff7c87..55a5a143bc245 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/enrich/AbstractLookupService.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/enrich/AbstractLookupService.java @@ -198,6 +198,7 @@ protected static QueryList termQueryList( return switch (inputDataType) { case IP -> QueryList.ipTermQueryList(field, searchExecutionContext, (BytesRefBlock) block); case DATETIME -> QueryList.dateTermQueryList(field, searchExecutionContext, (LongBlock) block); + case DATE_NANOS -> QueryList.dateNanosTermQueryList(field, searchExecutionContext, (LongBlock) block); case null, default -> QueryList.rawTermQueryList(field, searchExecutionContext, block); }; } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/join/Join.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/join/Join.java index 175d110c7ab51..df89bc85273cb 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/join/Join.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/join/Join.java @@ -37,7 +37,6 @@ import static org.elasticsearch.xpack.esql.core.type.DataType.COUNTER_DOUBLE; import static org.elasticsearch.xpack.esql.core.type.DataType.COUNTER_INTEGER; import static org.elasticsearch.xpack.esql.core.type.DataType.COUNTER_LONG; -import static org.elasticsearch.xpack.esql.core.type.DataType.DATE_NANOS; import static org.elasticsearch.xpack.esql.core.type.DataType.DATE_PERIOD; import static org.elasticsearch.xpack.esql.core.type.DataType.DOC_DATA_TYPE; import static org.elasticsearch.xpack.esql.core.type.DataType.GEO_POINT; @@ -71,7 +70,6 @@ public class Join extends BinaryPlan implements PostAnalysisVerificationAware, S COUNTER_LONG, COUNTER_INTEGER, COUNTER_DOUBLE, - DATE_NANOS, OBJECT, SOURCE, DATE_PERIOD,