From cf5e3ee3ccd509286f3235aa04f530e5afa27c71 Mon Sep 17 00:00:00 2001 From: Craig Taverner Date: Wed, 7 May 2025 13:14:58 +0200 Subject: [PATCH 1/9] Integration tests for LOOKUP JOIN over wider range of data types (#126150) This test suite tests the lookup join functionality in ESQL with various data types. For each pair of types being tested, it builds a main index called "index" containing a single document with as many fields as types being tested on the left of the pair, and then creates that many other lookup indexes, each with a single document containing exactly two fields: the field to join on, and a field to return. The assertion is that for valid combinations, the return result should exist, and for invalid combinations an exception should be thrown. If no exception is thrown, and no result is returned, our validation rules are not aligned with the internal behaviour (ie. a bug). Since the `LOOKUP JOIN` command requires the match field name to be the same between the main index and the lookup index, we will have field names that correctly represent the type of the field in the main index, but not the type of the field in the lookup index. This can be confusing, but it is important to remember that the field names are not the same as the types. --- x-pack/plugin/esql/build.gradle | 1 + .../xpack/esql/action/LookupJoinTypesIT.java | 542 ++++++++++++++++++ .../xpack/esql/plan/logical/join/Join.java | 55 +- 3 files changed, 596 insertions(+), 2 deletions(-) create mode 100644 x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/LookupJoinTypesIT.java diff --git a/x-pack/plugin/esql/build.gradle b/x-pack/plugin/esql/build.gradle index 2310de66ae86b..a0793635dc315 100644 --- a/x-pack/plugin/esql/build.gradle +++ b/x-pack/plugin/esql/build.gradle @@ -46,6 +46,7 @@ dependencies { testImplementation project(path: xpackModule('enrich')) testImplementation project(path: xpackModule('spatial')) testImplementation project(path: xpackModule('kql')) + testImplementation project(path: xpackModule('mapper-unsigned-long')) testImplementation project(path: ':modules:reindex') testImplementation project(path: ':modules:parent-join') diff --git a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/LookupJoinTypesIT.java b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/LookupJoinTypesIT.java new file mode 100644 index 0000000000000..52c41e4056a8e --- /dev/null +++ b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/LookupJoinTypesIT.java @@ -0,0 +1,542 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.action; + +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.index.mapper.extras.MapperExtrasPlugin; +import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.test.ESIntegTestCase; +import org.elasticsearch.test.ESIntegTestCase.ClusterScope; +import org.elasticsearch.xpack.core.esql.action.ColumnInfo; +import org.elasticsearch.xpack.esql.VerificationException; +import org.elasticsearch.xpack.esql.core.type.DataType; +import org.elasticsearch.xpack.esql.plan.logical.join.Join; +import org.elasticsearch.xpack.esql.plugin.EsqlPlugin; +import org.elasticsearch.xpack.spatial.SpatialPlugin; +import org.elasticsearch.xpack.unsignedlong.UnsignedLongMapperPlugin; +import org.elasticsearch.xpack.versionfield.VersionFieldPlugin; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Iterator; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Set; +import java.util.function.Consumer; +import java.util.stream.Collectors; + +import static org.elasticsearch.test.ESIntegTestCase.Scope.SUITE; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; +import static org.elasticsearch.xpack.esql.core.type.DataType.AGGREGATE_METRIC_DOUBLE; +import static org.elasticsearch.xpack.esql.core.type.DataType.BOOLEAN; +import static org.elasticsearch.xpack.esql.core.type.DataType.BYTE; +import static org.elasticsearch.xpack.esql.core.type.DataType.DATETIME; +import static org.elasticsearch.xpack.esql.core.type.DataType.DOC_DATA_TYPE; +import static org.elasticsearch.xpack.esql.core.type.DataType.DOUBLE; +import static org.elasticsearch.xpack.esql.core.type.DataType.FLOAT; +import static org.elasticsearch.xpack.esql.core.type.DataType.HALF_FLOAT; +import static org.elasticsearch.xpack.esql.core.type.DataType.INTEGER; +import static org.elasticsearch.xpack.esql.core.type.DataType.IP; +import static org.elasticsearch.xpack.esql.core.type.DataType.KEYWORD; +import static org.elasticsearch.xpack.esql.core.type.DataType.LONG; +import static org.elasticsearch.xpack.esql.core.type.DataType.NULL; +import static org.elasticsearch.xpack.esql.core.type.DataType.SCALED_FLOAT; +import static org.elasticsearch.xpack.esql.core.type.DataType.SHORT; +import static org.elasticsearch.xpack.esql.core.type.DataType.TEXT; +import static org.elasticsearch.xpack.esql.core.type.DataType.TSID_DATA_TYPE; +import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.nullValue; + +/** + * This test suite tests the lookup join functionality in ESQL with various data types. + * For each pair of types being tested, it builds a main index called "index" containing a single document with as many fields as + * types being tested on the left of the pair, and then creates that many other lookup indexes, each with a single document containing + * exactly two fields: the field to join on, and a field to return. + * The assertion is that for valid combinations, the return result should exist, and for invalid combinations an exception should be thrown. + * If no exception is thrown, and no result is returned, our validation rules are not aligned with the internal behaviour (ie. a bug). + * Since the `LOOKUP JOIN` command requires the match field name to be the same between the main index and the lookup index, + * we will have field names that correctly represent the type of the field in the main index, but not the type of the field + * in the lookup index. This can be confusing, but it is important to remember that the field names are not the same as the types. + * For example, if we are testing the pairs (double, double), (double, float), (float, double) and (float, float), + * we will create the following indexes: + *
+ *
index_double_double
+ *
Index containing a single document with a field of type 'double' like: 
+ *         {
+ *             "field_double": 1.0,  // this is mapped as type 'double'
+ *             "other": "value"
+ *         }
+ *
+ *
index_double_float
+ *
Index containing a single document with a field of type 'float' like: 
+ *         {
+ *             "field_double": 1.0,  // this is mapped as type 'float' (a float with the name of the main index field)
+ *             "other": "value"
+ *         }
+ *
+ *
index_float_double
+ *
Index containing a single document with a field of type 'double' like: 
+ *         {
+ *             "field_float": 1.0,  // this is mapped as type 'double' (a double with the name of the main index field)
+ *             "other": "value"
+ *         }
+ *
+ *
index_float_float
+ *
Index containing single document with a field of type 'float' like: 
+ *         {
+ *             "field_float": 1.0,  // this is mapped as type 'float'
+ *             "other": "value"
+ *         }
+ *
+ *
index
+ *
Index containing document like: 
+ *         {
+ *             "field_double": 1.0,  // this is mapped as type 'double'
+ *             "field_float": 1.0    // this is mapped as type 'float'
+ *         }
+ *
+ *
+ * Note that the lookup indexes have fields with a name that matches the type in the main index, and not the type actually used in the + * lookup index. Instead, the mapped type should be the type of the right-hand side of the pair being tested. + * Then we can run queries like: + * 
+ *     FROM index | LOOKUP JOIN index_double_float ON field_double | KEEP other
+ *
+ * And assert that the result exists and is equal to "value". + */ +@ClusterScope(scope = SUITE, numClientNodes = 1, numDataNodes = 1) +public class LookupJoinTypesIT extends ESIntegTestCase { + protected Collection> nodePlugins() { + return List.of( + EsqlPlugin.class, + MapperExtrasPlugin.class, + VersionFieldPlugin.class, + UnsignedLongMapperPlugin.class, + SpatialPlugin.class + ); + } + + private static final Map testConfigurations = new HashMap<>(); + static { + // Initialize the test configurations for string tests + { + TestConfigs configs = testConfigurations.computeIfAbsent("strings", TestConfigs::new); + configs.addPasses(KEYWORD, KEYWORD); + configs.addPasses(TEXT, KEYWORD); + configs.addFailsUnsupported(KEYWORD, TEXT); + } + + // Test integer types + var integerTypes = List.of(BYTE, SHORT, INTEGER); + { + TestConfigs configs = testConfigurations.computeIfAbsent("integers", TestConfigs::new); + for (DataType mainType : integerTypes) { + for (DataType lookupType : integerTypes) { + configs.addPasses(mainType, lookupType); + } + // Long is currently treated differently in the validation, but we could consider changing that + configs.addFails(mainType, LONG); + configs.addFails(LONG, mainType); + } + } + + // Test float and double + var floatTypes = List.of(HALF_FLOAT, FLOAT, DOUBLE, SCALED_FLOAT); + { + TestConfigs configs = testConfigurations.computeIfAbsent("floats", TestConfigs::new); + for (DataType mainType : floatTypes) { + for (DataType lookupType : floatTypes) { + configs.addPasses(mainType, lookupType); + } + } + } + + // Tests for mixed-numerical types + { + TestConfigs configs = testConfigurations.computeIfAbsent("mixed-numerical", TestConfigs::new); + for (DataType mainType : integerTypes) { + for (DataType lookupType : floatTypes) { + // TODO: We should probably allow this, but we need to change the validation code in Join.java + configs.addFails(mainType, lookupType); + configs.addFails(lookupType, mainType); + } + } + } + + // Tests for all unsupported types + DataType[] unsupported = Join.UNSUPPORTED_TYPES; + { + Collection existing = testConfigurations.values(); + TestConfigs configs = testConfigurations.computeIfAbsent("unsupported", TestConfigs::new); + for (DataType type : unsupported) { + if (type == NULL + || type == DOC_DATA_TYPE + || type == TSID_DATA_TYPE + || type == AGGREGATE_METRIC_DOUBLE + || type.esType() == null + || type.isCounter() + || DataType.isRepresentable(type) == false) { + // Skip unmappable types, or types not supported in ES|QL in general + continue; + } + if (existingIndex(existing, type, type)) { + // Skip existing configurations + continue; + } + configs.addFailsUnsupported(type, type); + } + } + + // Tests for all types where left and right are the same type + DataType[] supported = { BOOLEAN, LONG, INTEGER, DOUBLE, SHORT, BYTE, FLOAT, HALF_FLOAT, DATETIME, IP, KEYWORD, SCALED_FLOAT }; + { + Collection existing = testConfigurations.values(); + TestConfigs configs = testConfigurations.computeIfAbsent("same", TestConfigs::new); + for (DataType type : supported) { + assertThat("Claiming supported for unsupported type: " + type, List.of(unsupported).contains(type), is(false)); + if (existingIndex(existing, type, type) == false) { + // Only add the configuration if it doesn't already exist + configs.addPasses(type, type); + } + } + } + + // Assert that unsupported types are not in the supported list + for (DataType type : unsupported) { + assertThat("Claiming supported for unsupported type: " + type, List.of(supported).contains(type), is(false)); + } + + // Assert that unsupported+supported covers all types: + List missing = new ArrayList<>(); + for (DataType type : DataType.values()) { + boolean isUnsupported = List.of(unsupported).contains(type); + boolean isSupported = List.of(supported).contains(type); + if (isUnsupported == false && isSupported == false) { + missing.add(type); + } + } + assertThat(missing + " are not in the supported or unsupported list", missing.size(), is(0)); + + // Tests for all other type combinations + { + Collection existing = testConfigurations.values(); + TestConfigs configs = testConfigurations.computeIfAbsent("others", TestConfigs::new); + for (DataType mainType : supported) { + for (DataType lookupType : supported) { + if (existingIndex(existing, mainType, lookupType) == false) { + // Only add the configuration if it doesn't already exist + configs.addFails(mainType, lookupType); + } + } + } + } + + // Make sure we have never added two configurations with the same index name + Set knownTypes = new HashSet<>(); + for (TestConfigs configs : testConfigurations.values()) { + for (TestConfig config : configs.configs.values()) { + if (knownTypes.contains(config.indexName())) { + throw new IllegalArgumentException("Duplicate index name: " + config.indexName()); + } + knownTypes.add(config.indexName()); + } + } + } + + private static boolean existingIndex(Collection existing, DataType mainType, DataType lookupType) { + String indexName = "index_" + mainType.esType() + "_" + lookupType.esType(); + return existing.stream().anyMatch(c -> c.exists(indexName)); + } + + public void testLookupJoinStrings() { + testLookupJoinTypes("strings"); + } + + public void testLookupJoinIntegers() { + testLookupJoinTypes("integers"); + } + + public void testLookupJoinFloats() { + testLookupJoinTypes("floats"); + } + + public void testLookupJoinMixedNumerical() { + testLookupJoinTypes("mixed-numerical"); + } + + public void testLookupJoinSame() { + testLookupJoinTypes("same"); + } + + public void testLookupJoinUnsupported() { + testLookupJoinTypes("unsupported"); + } + + public void testLookupJoinOthers() { + testLookupJoinTypes("others"); + } + + private void testLookupJoinTypes(String group) { + initIndexes(group); + initData(group); + for (TestConfig config : testConfigurations.get(group).configs.values()) { + String query = String.format( + Locale.ROOT, + "FROM index | LOOKUP JOIN %s ON %s | KEEP other", + config.indexName(), + config.fieldName() + ); + config.validateMainIndex(); + config.validateLookupIndex(); + config.testQuery(query); + } + } + + private void initIndexes(String group) { + Collection configs = testConfigurations.get(group).configs.values(); + String propertyPrefix = "{\n \"properties\" : {\n"; + String propertySuffix = " }\n}\n"; + // The main index will have many fields, one of each type to use in later type specific joins + String mainFields = propertyPrefix + configs.stream() + .map(TestConfig::mainPropertySpec) + .distinct() + .collect(Collectors.joining(",\n ")) + propertySuffix; + assertAcked(prepareCreate("index").setMapping(mainFields)); + + Settings.Builder settings = Settings.builder() + .put("index.number_of_shards", 1) + .put("index.number_of_replicas", 0) + .put("index.mode", "lookup"); + configs.forEach( + // Each lookup index will get a document with a field to join on, and a results field to get back + (c) -> assertAcked( + prepareCreate(c.indexName()).setSettings(settings.build()) + .setMapping(propertyPrefix + c.lookupPropertySpec() + propertySuffix) + ) + ); + } + + private void initData(String group) { + Collection configs = testConfigurations.get(group).configs.values(); + int docId = 0; + for (TestConfig config : configs) { + String doc = String.format(Locale.ROOT, """ + { + %s, + "other": "value" + } + """, lookupPropertyFor(config)); + index(config.indexName(), "" + (++docId), doc); + refresh(config.indexName()); + } + List mainProperties = configs.stream().map(this::mainPropertyFor).distinct().collect(Collectors.toList()); + index("index", "1", String.format(Locale.ROOT, """ + { + %s + } + """, String.join(",\n ", mainProperties))); + refresh("index"); + } + + private String lookupPropertyFor(TestConfig config) { + return String.format(Locale.ROOT, "\"%s\": %s", config.fieldName(), sampleDataTextFor(config.lookupType())); + } + + private String mainPropertyFor(TestConfig config) { + return String.format(Locale.ROOT, "\"%s\": %s", config.fieldName(), sampleDataTextFor(config.mainType())); + } + + private static String sampleDataTextFor(DataType type) { + var value = sampleDataFor(type); + if (value instanceof String) { + return "\"" + value + "\""; + } + return String.valueOf(value); + } + + private static final double SCALING_FACTOR = 10.0; + + private static Object sampleDataFor(DataType type) { + return switch (type) { + case BOOLEAN -> true; + case DATETIME, DATE_NANOS -> "2025-04-02T12:00:00.000Z"; + case IP -> "127.0.0.1"; + case KEYWORD, TEXT -> "key"; + case BYTE, SHORT, INTEGER -> 1; + case LONG, UNSIGNED_LONG -> 1L; + case HALF_FLOAT, FLOAT, DOUBLE, SCALED_FLOAT -> 1.0; + case VERSION -> "1.2.19"; + case GEO_POINT, CARTESIAN_POINT -> "POINT (1.0 2.0)"; + case GEO_SHAPE, CARTESIAN_SHAPE -> "POLYGON ((0.0 0.0, 1.0 0.0, 1.0 1.0, 0.0 1.0, 0.0 0.0))"; + default -> throw new IllegalArgumentException("Unsupported type: " + type); + }; + } + + private static class TestConfigs { + final String group; + final Map configs; + + TestConfigs(String group) { + this.group = group; + this.configs = new LinkedHashMap<>(); + } + + private boolean exists(String indexName) { + return configs.containsKey(indexName); + } + + private void add(TestConfig config) { + if (configs.containsKey(config.indexName())) { + throw new IllegalArgumentException("Duplicate index name: " + config.indexName()); + } + configs.put(config.indexName(), config); + } + + private void addPasses(DataType mainType, DataType lookupType) { + add(new TestConfigPasses(mainType, lookupType, true)); + } + + private void addFails(DataType mainType, DataType lookupType) { + String fieldName = "field_" + mainType.esType(); + String errorMessage = String.format( + Locale.ROOT, + "JOIN left field [%s] of type [%s] is incompatible with right field [%s] of type [%s]", + fieldName, + mainType.widenSmallNumeric(), + fieldName, + lookupType.widenSmallNumeric() + ); + add( + new TestConfigFails<>( + mainType, + lookupType, + VerificationException.class, + e -> assertThat(e.getMessage(), containsString(errorMessage)) + ) + ); + } + + private void addFailsUnsupported(DataType mainType, DataType lookupType) { + String fieldName = "field_" + mainType.esType(); + String errorMessage = String.format( + Locale.ROOT, + "JOIN with right field [%s] of type [%s] is not supported", + fieldName, + lookupType + ); + add( + new TestConfigFails<>( + mainType, + lookupType, + VerificationException.class, + e -> assertThat(e.getMessage(), containsString(errorMessage)) + ) + ); + } + } + + interface TestConfig { + DataType mainType(); + + DataType lookupType(); + + default String indexName() { + return "index_" + mainType().esType() + "_" + lookupType().esType(); + } + + default String fieldName() { + return "field_" + mainType().esType(); + } + + default String mainPropertySpec() { + return propertySpecFor(fieldName(), mainType(), ""); + } + + default String lookupPropertySpec() { + return propertySpecFor(fieldName(), lookupType(), ", \"other\": { \"type\" : \"keyword\" }"); + } + + /** Make sure the left index has the expected fields and types */ + default void validateMainIndex() { + validateIndex("index", fieldName(), sampleDataFor(mainType())); + } + + /** Make sure the lookup index has the expected fields and types */ + default void validateLookupIndex() { + validateIndex(indexName(), fieldName(), sampleDataFor(lookupType())); + } + + void testQuery(String query); + } + + private static String propertySpecFor(String fieldName, DataType type, String extra) { + if (type == SCALED_FLOAT) { + return String.format( + Locale.ROOT, + "\"%s\": { \"type\" : \"%s\", \"scaling_factor\": %f }", + fieldName, + type.esType(), + SCALING_FACTOR + ) + extra; + } + return String.format(Locale.ROOT, "\"%s\": { \"type\" : \"%s\" }", fieldName, type.esType().replaceAll("cartesian_", "")) + extra; + } + + private static void validateIndex(String indexName, String fieldName, Object expectedValue) { + String query = String.format(Locale.ROOT, "FROM %s | KEEP %s", indexName, fieldName); + try (var response = EsqlQueryRequestBuilder.newRequestBuilder(client()).query(query).get()) { + ColumnInfo info = response.response().columns().getFirst(); + assertThat("Expected index '" + indexName + "' to have column '" + fieldName + ": " + query, info.name(), is(fieldName)); + Iterator results = response.response().column(0).iterator(); + assertTrue("Expected at least one result for query: " + query, results.hasNext()); + Object indexedResult = response.response().column(0).iterator().next(); + assertThat("Expected valid result: " + query, indexedResult, is(expectedValue)); + } + } + + private record TestConfigPasses(DataType mainType, DataType lookupType, boolean hasResults) implements TestConfig { + @Override + public void testQuery(String query) { + try (var response = EsqlQueryRequestBuilder.newRequestBuilder(client()).query(query).get()) { + Iterator results = response.response().column(0).iterator(); + assertTrue("Expected at least one result for query: " + query, results.hasNext()); + Object indexedResult = response.response().column(0).iterator().next(); + if (hasResults) { + assertThat("Expected valid result: " + query, indexedResult, equalTo("value")); + } else { + assertThat("Expected empty results for query: " + query, indexedResult, is(nullValue())); + } + } + } + } + + private record TestConfigFails(DataType mainType, DataType lookupType, Class exception, Consumer assertion) + implements + TestConfig { + @Override + public void testQuery(String query) { + E e = expectThrows( + exception(), + "Expected exception " + exception().getSimpleName() + " but no exception was thrown: " + query, + () -> { + // noinspection EmptyTryBlock + try (var ignored = EsqlQueryRequestBuilder.newRequestBuilder(client()).query(query).get()) { + // We use try-with-resources to ensure the request is closed if the exception is not thrown (less cluttered errors) + } + } + ); + assertion().accept(e); + } + } +} diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/join/Join.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/join/Join.java index 14877abb62272..af8a32d22ca99 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/join/Join.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/join/Join.java @@ -18,6 +18,7 @@ import org.elasticsearch.xpack.esql.core.expression.ReferenceAttribute; import org.elasticsearch.xpack.esql.core.tree.NodeInfo; import org.elasticsearch.xpack.esql.core.tree.Source; +import org.elasticsearch.xpack.esql.core.type.DataType; import org.elasticsearch.xpack.esql.io.stream.PlanStreamInput; import org.elasticsearch.xpack.esql.plan.logical.BinaryPlan; import org.elasticsearch.xpack.esql.plan.logical.LogicalPlan; @@ -25,16 +26,59 @@ import java.io.IOException; import java.util.ArrayList; +import java.util.Arrays; import java.util.List; import java.util.Objects; import static org.elasticsearch.xpack.esql.common.Failure.fail; +import static org.elasticsearch.xpack.esql.core.type.DataType.AGGREGATE_METRIC_DOUBLE; +import static org.elasticsearch.xpack.esql.core.type.DataType.CARTESIAN_POINT; +import static org.elasticsearch.xpack.esql.core.type.DataType.CARTESIAN_SHAPE; +import static org.elasticsearch.xpack.esql.core.type.DataType.COUNTER_DOUBLE; +import static org.elasticsearch.xpack.esql.core.type.DataType.COUNTER_INTEGER; +import static org.elasticsearch.xpack.esql.core.type.DataType.COUNTER_LONG; +import static org.elasticsearch.xpack.esql.core.type.DataType.DATE_NANOS; +import static org.elasticsearch.xpack.esql.core.type.DataType.DATE_PERIOD; +import static org.elasticsearch.xpack.esql.core.type.DataType.DOC_DATA_TYPE; +import static org.elasticsearch.xpack.esql.core.type.DataType.GEO_POINT; +import static org.elasticsearch.xpack.esql.core.type.DataType.GEO_SHAPE; +import static org.elasticsearch.xpack.esql.core.type.DataType.NULL; +import static org.elasticsearch.xpack.esql.core.type.DataType.OBJECT; +import static org.elasticsearch.xpack.esql.core.type.DataType.PARTIAL_AGG; +import static org.elasticsearch.xpack.esql.core.type.DataType.SOURCE; import static org.elasticsearch.xpack.esql.core.type.DataType.TEXT; +import static org.elasticsearch.xpack.esql.core.type.DataType.TIME_DURATION; +import static org.elasticsearch.xpack.esql.core.type.DataType.TSID_DATA_TYPE; +import static org.elasticsearch.xpack.esql.core.type.DataType.UNSIGNED_LONG; +import static org.elasticsearch.xpack.esql.core.type.DataType.UNSUPPORTED; +import static org.elasticsearch.xpack.esql.core.type.DataType.VERSION; import static org.elasticsearch.xpack.esql.expression.NamedExpressions.mergeOutputAttributes; import static org.elasticsearch.xpack.esql.plan.logical.join.JoinTypes.LEFT; public class Join extends BinaryPlan implements PostAnalysisVerificationAware, SortAgnostic { public static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry(LogicalPlan.class, "Join", Join::new); + public static final DataType[] UNSUPPORTED_TYPES = { + TEXT, + VERSION, + UNSIGNED_LONG, + GEO_POINT, + GEO_SHAPE, + CARTESIAN_POINT, + CARTESIAN_SHAPE, + UNSUPPORTED, + NULL, + COUNTER_LONG, + COUNTER_INTEGER, + COUNTER_DOUBLE, + DATE_NANOS, + OBJECT, + SOURCE, + DATE_PERIOD, + TIME_DURATION, + DOC_DATA_TYPE, + TSID_DATA_TYPE, + PARTIAL_AGG, + AGGREGATE_METRIC_DOUBLE }; private final JoinConfig config; private List lazyOutput; @@ -213,7 +257,7 @@ public void postAnalysisVerification(Failures failures) { for (int i = 0; i < config.leftFields().size(); i++) { Attribute leftField = config.leftFields().get(i); Attribute rightField = config.rightFields().get(i); - if (leftField.dataType().noText() != rightField.dataType().noText()) { + if (comparableTypes(leftField, rightField) == false) { failures.add( fail( leftField, @@ -225,11 +269,18 @@ public void postAnalysisVerification(Failures failures) { ) ); } - if (rightField.dataType().equals(TEXT)) { + // TODO: Add support for VERSION by implementing QueryList.versionTermQueryList similar to ipTermQueryList + if (Arrays.stream(UNSUPPORTED_TYPES).anyMatch(t -> rightField.dataType().equals(t))) { failures.add( fail(leftField, "JOIN with right field [{}] of type [{}] is not supported", rightField.name(), rightField.dataType()) ); } } } + + private static boolean comparableTypes(Attribute left, Attribute right) { + // TODO: Consider allowing more valid types + // return left.dataType().noText() == right.dataType().noText() || left.dataType().isNumeric() == right.dataType().isNumeric(); + return left.dataType().noText() == right.dataType().noText(); + } } From 1cb31314fd1e1fb24c020cccb8ad3611e48af745 Mon Sep 17 00:00:00 2001 From: Craig Taverner Date: Mon, 9 Jun 2025 11:16:23 +0200 Subject: [PATCH 2/9] Just use one lookup-settings file This change simplifies backports from 9.x branches where these changes were done as part of other work. --- .../elasticsearch/xpack/esql/CsvTestsDataLoader.java | 10 ++++------ .../src/main/resources/languages_lookup-settings.json | 5 ----- ...ntips_lookup-settings.json => lookup-settings.json} | 0 .../main/resources/message_types_lookup-settings.json | 5 ----- 4 files changed, 4 insertions(+), 16 deletions(-) delete mode 100644 x-pack/plugin/esql/qa/testFixtures/src/main/resources/languages_lookup-settings.json rename x-pack/plugin/esql/qa/testFixtures/src/main/resources/{clientips_lookup-settings.json => lookup-settings.json} (100%) delete mode 100644 x-pack/plugin/esql/qa/testFixtures/src/main/resources/message_types_lookup-settings.json diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/CsvTestsDataLoader.java b/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/CsvTestsDataLoader.java index 8ded542ee30b9..d6c0773995233 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/CsvTestsDataLoader.java +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/CsvTestsDataLoader.java @@ -61,15 +61,14 @@ public class CsvTestsDataLoader { private static final TestDataset APPS = new TestDataset("apps"); private static final TestDataset APPS_SHORT = APPS.withIndex("apps_short").withTypeMapping(Map.of("id", "short")); private static final TestDataset LANGUAGES = new TestDataset("languages"); - private static final TestDataset LANGUAGES_LOOKUP = LANGUAGES.withIndex("languages_lookup") - .withSetting("languages_lookup-settings.json"); + private static final TestDataset LANGUAGES_LOOKUP = LANGUAGES.withIndex("languages_lookup").withSetting("lookup-settings.json"); private static final TestDataset LANGUAGES_LOOKUP_NON_UNIQUE_KEY = LANGUAGES_LOOKUP.withIndex("languages_lookup_non_unique_key") .withData("languages_non_unique_key.csv"); private static final TestDataset LANGUAGES_NESTED_FIELDS = new TestDataset( "languages_nested_fields", "mapping-languages_nested_fields.json", "languages_nested_fields.csv" - ).withSetting("languages_lookup-settings.json"); + ).withSetting("lookup-settings.json"); private static final TestDataset ALERTS = new TestDataset("alerts"); private static final TestDataset UL_LOGS = new TestDataset("ul_logs"); private static final TestDataset SAMPLE_DATA = new TestDataset("sample_data"); @@ -84,11 +83,10 @@ public class CsvTestsDataLoader { .withTypeMapping(Map.of("@timestamp", "date_nanos")); private static final TestDataset MISSING_IP_SAMPLE_DATA = new TestDataset("missing_ip_sample_data"); private static final TestDataset CLIENT_IPS = new TestDataset("clientips"); - private static final TestDataset CLIENT_IPS_LOOKUP = CLIENT_IPS.withIndex("clientips_lookup") - .withSetting("clientips_lookup-settings.json"); + private static final TestDataset CLIENT_IPS_LOOKUP = CLIENT_IPS.withIndex("clientips_lookup").withSetting("lookup-settings.json"); private static final TestDataset MESSAGE_TYPES = new TestDataset("message_types"); private static final TestDataset MESSAGE_TYPES_LOOKUP = MESSAGE_TYPES.withIndex("message_types_lookup") - .withSetting("message_types_lookup-settings.json"); + .withSetting("lookup-settings.json"); private static final TestDataset CLIENT_CIDR = new TestDataset("client_cidr"); private static final TestDataset AGES = new TestDataset("ages"); private static final TestDataset HEIGHTS = new TestDataset("heights"); diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/languages_lookup-settings.json b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/languages_lookup-settings.json deleted file mode 100644 index b73d1f9accf92..0000000000000 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/languages_lookup-settings.json +++ /dev/null @@ -1,5 +0,0 @@ -{ - "index": { - "mode": "lookup" - } -} diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/clientips_lookup-settings.json b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/lookup-settings.json similarity index 100% rename from x-pack/plugin/esql/qa/testFixtures/src/main/resources/clientips_lookup-settings.json rename to x-pack/plugin/esql/qa/testFixtures/src/main/resources/lookup-settings.json diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/message_types_lookup-settings.json b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/message_types_lookup-settings.json deleted file mode 100644 index b73d1f9accf92..0000000000000 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/message_types_lookup-settings.json +++ /dev/null @@ -1,5 +0,0 @@ -{ - "index": { - "mode": "lookup" - } -} From a7cf667e75d08d1463289e7080ef96f1ce23c6c0 Mon Sep 17 00:00:00 2001 From: Craig Taverner Date: Wed, 4 Jun 2025 23:25:39 +0200 Subject: [PATCH 3/9] Support DATE_NANOS in LOOKUP JOIN (#127962) We reported in https://github.com/elastic/elasticsearch/issues/127249, there is no support for DATE_NANOS in LOOKUP JOIN, even though DATETIME is supported. This PR attempts to fix that. The way that date-time was supported in LOOKUP JOIN (and ENRICH) was by using the `DateFieldMapper.DateFieldType.rangeQuery` (hidden behind the `termQuery` function) which internally takes our long values, casts them to Object, renders them to a string, parses that string back into an Instant (with a bunch of fancy and unnecessary checks for date-math, etc.), and then converts that instant back into a long for the actual query. Parts of this complex process are precision aware (ie. differentiate between ms and ns dates), but not the whole process. Simply dividing the original longs by 1_000_000 before passing them in actually works, but obviously looses precision. And the only reason it works anyway is that the date parsing code will accept a string containing a simple number and interpret it as either ms since the epoch, or years if the number is short enough. This does not work for nano-second dates, and in fact is far from ideal for LOOKUP JOIN on dates which does not need to re-parse the values at all. This complex loop only makes sense in the Query DSL, where we can get all kinds of interesting sources of range values, but seems quite crazy for LOOKUP JOIN where we will always provide the join key from a LongBlock (the backing store of the DATE_TIME DataType, and the DATE_NANOS too). So what we do here for DateNanos is provide two new methods to `DateFieldType`: * `equalityQuery(Long, ...)` to replace `termQuery(Object, ...)` * `rangeQuery(Long, Long, ...)` to replace `rangeQuery(Object, Object, ...)` This allows us to pass in already parsed `long` values, and entirely skip the conversion to strings and re-parsing logic. The new methods are based on the original methods, but considerably simplified due to the removal of the complex parsing logic. The reason for both `equalityQuery` and `rangeQuery` is that it mimics the pattern used by the old `termQuery` with delegated directly down to `rangeQuery`. In addition to this, we hope to support range matching in `LOOKUP JOIN` in the near future. --- docs/changelog/127962.yaml | 6 + .../index/mapper/DateFieldMapper.java | 56 +++- .../index/mapper/DateFieldTypeTests.java | 294 ++++++++++++++---- .../compute/operator/lookup/QueryList.java | 74 +++++ .../elasticsearch/xpack/esql/CsvAssert.java | 2 +- .../xpack/esql/CsvTestsDataLoader.java | 3 + .../src/main/resources/lookup-join.csv-spec | 24 ++ .../xpack/esql/action/LookupJoinTypesIT.java | 33 +- .../xpack/esql/action/EsqlCapabilities.java | 6 + .../esql/enrich/AbstractLookupService.java | 1 + .../xpack/esql/plan/logical/join/Join.java | 2 - 11 files changed, 441 insertions(+), 60 deletions(-) create mode 100644 docs/changelog/127962.yaml diff --git a/docs/changelog/127962.yaml b/docs/changelog/127962.yaml new file mode 100644 index 0000000000000..f96b79a69d5b3 --- /dev/null +++ b/docs/changelog/127962.yaml @@ -0,0 +1,6 @@ +pr: 127962 +summary: Support DATE_NANOS in LOOKUP JOIN +area: ES|QL +type: bug +issues: + - 127249 diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java index 5ddaff8b961bb..55d29d37ad478 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java @@ -473,7 +473,7 @@ public DateFieldType(String name) { this(name, true, true, false, true, DEFAULT_DATE_TIME_FORMATTER, Resolution.MILLISECONDS, null, null, Collections.emptyMap()); } - public DateFieldType(String name, boolean isIndexed) { + public DateFieldType(String name, boolean isIndexed, Resolution resolution) { this( name, isIndexed, @@ -481,13 +481,17 @@ public DateFieldType(String name, boolean isIndexed) { false, true, DEFAULT_DATE_TIME_FORMATTER, - Resolution.MILLISECONDS, + resolution, null, null, Collections.emptyMap() ); } + public DateFieldType(String name, boolean isIndexed) { + this(name, isIndexed, Resolution.MILLISECONDS); + } + public DateFieldType(String name, DateFormatter dateFormatter) { this(name, true, true, false, true, dateFormatter, Resolution.MILLISECONDS, null, null, Collections.emptyMap()); } @@ -699,6 +703,54 @@ public static long parseToLong( return resolution.convert(dateParser.parse(BytesRefs.toString(value), now, roundUp, zone)); } + /** + * Similar to the {@link DateFieldType#termQuery} method, but works on dates that are already parsed to a long + * in the same precision as the field mapper. + */ + public Query equalityQuery(Long value, @Nullable SearchExecutionContext context) { + return rangeQuery(value, value, true, true, context); + } + + /** + * Similar to the existing + * {@link DateFieldType#rangeQuery(Object, Object, boolean, boolean, ShapeRelation, ZoneId, DateMathParser, SearchExecutionContext)} + * method, but works on dates that are already parsed to a long in the same precision as the field mapper. + */ + public Query rangeQuery( + Long lowerTerm, + Long upperTerm, + boolean includeLower, + boolean includeUpper, + SearchExecutionContext context + ) { + failIfNotIndexedNorDocValuesFallback(context); + long l, u; + if (lowerTerm == null) { + l = Long.MIN_VALUE; + } else { + l = (includeLower == false) ? lowerTerm + 1 : lowerTerm; + } + if (upperTerm == null) { + u = Long.MAX_VALUE; + } else { + u = (includeUpper == false) ? upperTerm - 1 : upperTerm; + } + Query query; + if (isIndexed()) { + query = LongPoint.newRangeQuery(name(), l, u); + if (hasDocValues()) { + Query dvQuery = SortedNumericDocValuesField.newSlowRangeQuery(name(), l, u); + query = new IndexOrDocValuesQuery(query, dvQuery); + } + } else { + query = SortedNumericDocValuesField.newSlowRangeQuery(name(), l, u); + } + if (hasDocValues() && context.indexSortedOnField(name())) { + query = new XIndexSortSortedNumericDocValuesRangeQuery(name(), l, u, query); + } + return query; + } + @Override public Query distanceFeatureQuery(Object origin, String pivot, SearchExecutionContext context) { failIfNotIndexedNorDocValuesFallback(context); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/DateFieldTypeTests.java b/server/src/test/java/org/elasticsearch/index/mapper/DateFieldTypeTests.java index d925a9dd1d691..2f3781f63e5ee 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/DateFieldTypeTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/DateFieldTypeTests.java @@ -208,13 +208,11 @@ public void testValueForSearch() { assertEquals(date, ft.valueForDisplay(instant)); } + /** + * If the term field is a string of date-time format with exact seconds (no sub-seconds), any data within a 1second range will match. + */ public void testTermQuery() { - Settings indexSettings = indexSettings(IndexVersion.current(), 1, 1).build(); - SearchExecutionContext context = SearchExecutionContextHelper.createSimple( - new IndexSettings(IndexMetadata.builder("foo").settings(indexSettings).build(), indexSettings), - parserConfig(), - writableRegistry() - ); + SearchExecutionContext context = prepareIndexForTermQuery(); MappedFieldType ft = new DateFieldType("field"); String date = "2015-10-12T14:10:55"; long instant = DateFormatters.from(DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.parse(date)).toInstant().toEpochMilli(); @@ -228,45 +226,99 @@ public void testTermQuery() { expected = SortedNumericDocValuesField.newSlowRangeQuery("field", instant, instant + 999); assertEquals(expected, ft.termQuery(date, context)); - MappedFieldType unsearchable = new DateFieldType( - "field", - false, - false, - false, - DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER, - Resolution.MILLISECONDS, - null, - null, - Collections.emptyMap() + assertIndexUnsearchable(Resolution.MILLISECONDS, (unsearchable) -> unsearchable.termQuery(date, context)); + } + + /** + * If the term field is a string of date-time format with sub-seconds, only data with exact ms precision will match. + */ + public void testTermQuerySubseconds() { + SearchExecutionContext context = prepareIndexForTermQuery(); + MappedFieldType ft = new DateFieldType("field"); + String date = "2015-10-12T14:10:55.01"; + long instant = DateFormatters.from(DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.parse(date)).toInstant().toEpochMilli(); + Query expected = new IndexOrDocValuesQuery( + LongPoint.newRangeQuery("field", instant, instant), + SortedNumericDocValuesField.newSlowRangeQuery("field", instant, instant) ); - IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> unsearchable.termQuery(date, context)); - assertEquals("Cannot search on field [field] since it is not indexed nor has doc values.", e.getMessage()); + assertEquals(expected, ft.termQuery(date, context)); + + ft = new DateFieldType("field", false); + expected = SortedNumericDocValuesField.newSlowRangeQuery("field", instant, instant); + assertEquals(expected, ft.termQuery(date, context)); + + assertIndexUnsearchable(Resolution.MILLISECONDS, (unsearchable) -> unsearchable.termQuery(date, context)); } - public void testRangeQuery() throws IOException { - Settings indexSettings = indexSettings(IndexVersion.current(), 1, 1).build(); - SearchExecutionContext context = new SearchExecutionContext( - 0, - 0, - new IndexSettings(IndexMetadata.builder("foo").settings(indexSettings).build(), indexSettings), - null, - null, - null, - MappingLookup.EMPTY, - null, - null, - parserConfig(), - writableRegistry(), - null, - null, - () -> nowInMillis, - null, - null, - () -> true, - null, - Collections.emptyMap(), - MapperMetrics.NOOP + /** + * If the term field is a string of the long value (ms since epoch), only data with exact ms precision will match. + */ + public void testTermQueryMillis() { + SearchExecutionContext context = prepareIndexForTermQuery(); + MappedFieldType ft = new DateFieldType("field"); + String date = "2015-10-12T14:10:55"; + long instant = DateFormatters.from(DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.parse(date)).toInstant().toEpochMilli(); + Query expected = new IndexOrDocValuesQuery( + LongPoint.newRangeQuery("field", instant, instant), + SortedNumericDocValuesField.newSlowRangeQuery("field", instant, instant) ); + assertEquals(expected, ft.termQuery(instant, context)); + + ft = new DateFieldType("field", false); + expected = SortedNumericDocValuesField.newSlowRangeQuery("field", instant, instant); + assertEquals(expected, ft.termQuery(instant, context)); + + assertIndexUnsearchable(Resolution.MILLISECONDS, (unsearchable) -> unsearchable.termQuery(instant, context)); + } + + /** + * This query has similar behaviour to passing a String containing a long to termQuery, only data with exact ms precision will match. + */ + public void testEqualityQuery() { + SearchExecutionContext context = prepareIndexForTermQuery(); + DateFieldType ft = new DateFieldType("field"); + String date = "2015-10-12T14:10:55"; + long instant = DateFormatters.from(DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.parse(date)).toInstant().toEpochMilli(); + Query expected = new IndexOrDocValuesQuery( + LongPoint.newRangeQuery("field", instant, instant), + SortedNumericDocValuesField.newSlowRangeQuery("field", instant, instant) + ); + assertEquals(expected, ft.equalityQuery(instant, context)); + + ft = new DateFieldType("field", false); + expected = SortedNumericDocValuesField.newSlowRangeQuery("field", instant, instant); + assertEquals(expected, ft.equalityQuery(instant, context)); + + assertIndexUnsearchable(Resolution.MILLISECONDS, (unsearchable) -> unsearchable.equalityQuery(instant, context)); + } + + /** + * This query supports passing a ns value, and only data with exact ns precision will match. + */ + public void testEqualityNanosQuery() { + SearchExecutionContext context = prepareIndexForTermQuery(); + DateFieldType ft = new DateFieldType("field", Resolution.NANOSECONDS); + String date = "2015-10-12T14:10:55"; + long instant = DateFormatters.from(DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.parse(date)).toInstant().toEpochMilli() * 1000000L; + Query expected = new IndexOrDocValuesQuery( + LongPoint.newRangeQuery("field", instant, instant), + SortedNumericDocValuesField.newSlowRangeQuery("field", instant, instant) + ); + assertEquals(expected, ft.equalityQuery(instant, context)); + + ft = new DateFieldType("field", false); + expected = SortedNumericDocValuesField.newSlowRangeQuery("field", instant, instant); + assertEquals(expected, ft.equalityQuery(instant, context)); + + assertIndexUnsearchable(Resolution.NANOSECONDS, (unsearchable) -> unsearchable.equalityQuery(instant, context)); + } + + /** + * If the term fields are strings of date-time format with exact seconds (no sub-seconds), + * the second field will be rounded up to the next second. + */ + public void testRangeQuery() throws IOException { + SearchExecutionContext context = prepareIndexForRangeQuery(); MappedFieldType ft = new DateFieldType("field"); String date1 = "2015-10-12T14:10:55"; String date2 = "2016-04-28T11:33:52"; @@ -298,22 +350,105 @@ public void testRangeQuery() throws IOException { expected2 = new DateRangeIncludingNowQuery(SortedNumericDocValuesField.newSlowRangeQuery("field", instant1, instant2)); assertEquals(expected2, ft2.rangeQuery("now", instant2, true, true, null, null, null, context)); - MappedFieldType unsearchable = new DateFieldType( - "field", - false, - false, - false, - DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER, + assertIndexUnsearchable( Resolution.MILLISECONDS, - null, - null, - Collections.emptyMap() + (unsearchable) -> unsearchable.rangeQuery(date1, date2, true, true, null, null, null, context) ); - IllegalArgumentException e = expectThrows( - IllegalArgumentException.class, - () -> unsearchable.rangeQuery(date1, date2, true, true, null, null, null, context) + } + + /** + * If the term fields are strings of date-time format with sub-seconds, + * the lower and upper values will be matched inclusively to the ms. + */ + public void testRangeQuerySubseconds() throws IOException { + SearchExecutionContext context = prepareIndexForRangeQuery(); + MappedFieldType ft = new DateFieldType("field"); + String date1 = "2015-10-12T14:10:55.01"; + String date2 = "2016-04-28T11:33:52.01"; + long instant1 = DateFormatters.from(DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.parse(date1)).toInstant().toEpochMilli(); + long instant2 = DateFormatters.from(DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.parse(date2)).toInstant().toEpochMilli(); + Query expected = new IndexOrDocValuesQuery( + LongPoint.newRangeQuery("field", instant1, instant2), + SortedNumericDocValuesField.newSlowRangeQuery("field", instant1, instant2) + ); + assertEquals(expected, ft.rangeQuery(date1, date2, true, true, null, null, null, context).rewrite(newSearcher(new MultiReader()))); + + MappedFieldType ft2 = new DateFieldType("field", false); + Query expected2 = SortedNumericDocValuesField.newSlowRangeQuery("field", instant1, instant2); + assertEquals( + expected2, + ft2.rangeQuery(date1, date2, true, true, null, null, null, context).rewrite(newSearcher(new MultiReader())) + ); + + instant1 = nowInMillis; + instant2 = instant1 + 100; + expected = new DateRangeIncludingNowQuery( + new IndexOrDocValuesQuery( + LongPoint.newRangeQuery("field", instant1, instant2), + SortedNumericDocValuesField.newSlowRangeQuery("field", instant1, instant2) + ) + ); + assertEquals(expected, ft.rangeQuery("now", instant2, true, true, null, null, null, context)); + + expected2 = new DateRangeIncludingNowQuery(SortedNumericDocValuesField.newSlowRangeQuery("field", instant1, instant2)); + assertEquals(expected2, ft2.rangeQuery("now", instant2, true, true, null, null, null, context)); + + assertIndexUnsearchable( + Resolution.MILLISECONDS, + (unsearchable) -> unsearchable.rangeQuery(date1, date2, true, true, null, null, null, context) ); - assertEquals("Cannot search on field [field] since it is not indexed nor has doc values.", e.getMessage()); + } + + /** + * If the term fields are strings of long ms, the lower and upper values will be matched inclusively to the ms. + */ + public void testRangeQueryMillis() throws IOException { + SearchExecutionContext context = prepareIndexForRangeQuery(); + DateFieldType ft = new DateFieldType("field"); + String date1 = "2015-10-12T14:10:55.01"; + String date2 = "2016-04-28T11:33:52.01"; + long instant1 = DateFormatters.from(DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.parse(date1)).toInstant().toEpochMilli(); + long instant2 = DateFormatters.from(DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.parse(date2)).toInstant().toEpochMilli(); + Query expected = new IndexOrDocValuesQuery( + LongPoint.newRangeQuery("field", instant1, instant2), + SortedNumericDocValuesField.newSlowRangeQuery("field", instant1, instant2) + ); + assertEquals(expected, ft.rangeQuery(instant1, instant2, true, true, context).rewrite(newSearcher(new MultiReader()))); + + DateFieldType ft2 = new DateFieldType("field", false); + Query expected2 = SortedNumericDocValuesField.newSlowRangeQuery("field", instant1, instant2); + assertEquals(expected2, ft2.rangeQuery(instant1, instant2, true, true, context).rewrite(newSearcher(new MultiReader()))); + + assertIndexUnsearchable( + Resolution.MILLISECONDS, + (unsearchable) -> unsearchable.rangeQuery(instant1, instant2, true, true, context) + ); + } + + /** + * If the term fields are strings of long ns, the lower and upper values will be matched inclusively to the ns. + */ + public void testRangeQueryNanos() throws IOException { + SearchExecutionContext context = prepareIndexForRangeQuery(); + DateFieldType ft = new DateFieldType("field", Resolution.NANOSECONDS); + String date1 = "2015-10-12T14:10:55.01"; + String date2 = "2016-04-28T11:33:52.01"; + long instant1 = DateFormatters.from(DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.parse(date1)).toInstant().toEpochMilli() * 1000000L; + long instant2 = DateFormatters.from(DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.parse(date2)).toInstant().toEpochMilli() * 1000000L; + Query expected = new IndexOrDocValuesQuery( + LongPoint.newRangeQuery("field", instant1, instant2), + SortedNumericDocValuesField.newSlowRangeQuery("field", instant1, instant2) + ); + assertEquals(expected, ft.rangeQuery(instant1, instant2, true, true, context).rewrite(newSearcher(new MultiReader()))); + + DateFieldType ft2 = new DateFieldType("field", false, Resolution.NANOSECONDS); + Query expected2 = SortedNumericDocValuesField.newSlowRangeQuery("field", instant1, instant2); + assertEquals( + expected2, + ft2.rangeQuery(date1, date2, true, true, null, null, null, context).rewrite(newSearcher(new MultiReader())) + ); + + assertIndexUnsearchable(Resolution.NANOSECONDS, (unsearchable) -> unsearchable.rangeQuery(instant1, instant2, true, true, context)); } public void testRangeQueryWithIndexSort() { @@ -420,4 +555,55 @@ public void testParseSourceValueNanos() throws IOException { MappedFieldType nullValueMapper = fieldType(Resolution.NANOSECONDS, "strict_date_time||epoch_millis", nullValueDate); assertEquals(List.of(nullValueDate), fetchSourceValue(nullValueMapper, null)); } + + private SearchExecutionContext prepareIndexForTermQuery() { + Settings indexSettings = indexSettings(IndexVersion.current(), 1, 1).build(); + return SearchExecutionContextHelper.createSimple( + new IndexSettings(IndexMetadata.builder("foo").settings(indexSettings).build(), indexSettings), + parserConfig(), + writableRegistry() + ); + } + + private SearchExecutionContext prepareIndexForRangeQuery() { + Settings indexSettings = indexSettings(IndexVersion.current(), 1, 1).build(); + return new SearchExecutionContext( + 0, + 0, + new IndexSettings(IndexMetadata.builder("foo").settings(indexSettings).build(), indexSettings), + null, + null, + null, + MappingLookup.EMPTY, + null, + null, + parserConfig(), + writableRegistry(), + null, + null, + () -> nowInMillis, + null, + null, + () -> true, + null, + Collections.emptyMap(), + MapperMetrics.NOOP + ); + } + + private void assertIndexUnsearchable(Resolution resolution, ThrowingConsumer runnable) { + DateFieldType unsearchable = new DateFieldType( + "field", + false, + false, + false, + DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER, + resolution, + null, + null, + Collections.emptyMap() + ); + IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> runnable.accept(unsearchable)); + assertEquals("Cannot search on field [field] since it is not indexed nor has doc values.", e.getMessage()); + } } diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/lookup/QueryList.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/lookup/QueryList.java index 5d359e2fb612f..d6fb6e4bfcdec 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/lookup/QueryList.java +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/lookup/QueryList.java @@ -11,6 +11,7 @@ import org.apache.lucene.geo.GeoEncodingUtils; import org.apache.lucene.search.BooleanClause; import org.apache.lucene.search.BooleanQuery; +import org.apache.lucene.search.ConstantScoreQuery; import org.apache.lucene.search.MatchAllDocsQuery; import org.apache.lucene.search.Query; import org.apache.lucene.util.BytesRef; @@ -30,6 +31,7 @@ import org.elasticsearch.geometry.Point; import org.elasticsearch.geometry.utils.GeometryValidator; import org.elasticsearch.geometry.utils.WellKnownBinary; +import org.elasticsearch.index.mapper.DateFieldMapper; import org.elasticsearch.index.mapper.GeoShapeQueryable; import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.RangeFieldMapper; @@ -37,7 +39,9 @@ import java.io.IOException; import java.util.ArrayList; +import java.util.HashSet; import java.util.List; +import java.util.Set; import java.util.function.IntFunction; /** @@ -189,6 +193,14 @@ public static QueryList dateTermQueryList(MappedFieldType field, SearchExecution ); } + /** + * Returns a list of term queries for the given field and the input block of + * {@code date_nanos} field values. + */ + public static QueryList dateNanosTermQueryList(MappedFieldType field, SearchExecutionContext searchExecutionContext, LongBlock block) { + return new DateNanosQueryList(field, searchExecutionContext, block, false); + } + /** * Returns a list of geo_shape queries for the given field and the input block. */ @@ -232,6 +244,68 @@ Query doGetQuery(int position, int firstValueIndex, int valueCount) { } } + private static class DateNanosQueryList extends QueryList { + protected final IntFunction blockValueReader; + private final DateFieldMapper.DateFieldType dateFieldType; + + private DateNanosQueryList( + MappedFieldType field, + SearchExecutionContext searchExecutionContext, + LongBlock block, + boolean onlySingleValues + ) { + super(field, searchExecutionContext, block, onlySingleValues); + if (field instanceof RangeFieldMapper.RangeFieldType rangeFieldType) { + // TODO: do this validation earlier + throw new IllegalArgumentException( + "DateNanosQueryList does not support range fields [" + rangeFieldType + "]: " + field.name() + ); + } + this.blockValueReader = block::getLong; + if (field instanceof DateFieldMapper.DateFieldType dateFieldType) { + // Validate that the field is a date_nanos field + // TODO: Consider allowing date_nanos to match normal datetime fields + if (dateFieldType.resolution() != DateFieldMapper.Resolution.NANOSECONDS) { + throw new IllegalArgumentException( + "DateNanosQueryList only supports date_nanos fields, but got: " + field.typeName() + " for field: " + field.name() + ); + } + this.dateFieldType = dateFieldType; + } else { + throw new IllegalArgumentException( + "DateNanosQueryList only supports date_nanos fields, but got: " + field.typeName() + " for field: " + field.name() + ); + } + } + + @Override + public DateNanosQueryList onlySingleValues() { + return new DateNanosQueryList(field, searchExecutionContext, (LongBlock) block, true); + } + + @Override + Query doGetQuery(int position, int firstValueIndex, int valueCount) { + return switch (valueCount) { + case 0 -> null; + case 1 -> dateFieldType.equalityQuery(blockValueReader.apply(firstValueIndex), searchExecutionContext); + default -> { + // The following code is a slight simplification of the DateFieldMapper.termsQuery method + final Set values = new HashSet<>(valueCount); + BooleanQuery.Builder builder = new BooleanQuery.Builder(); + for (int i = 0; i < valueCount; i++) { + final Long value = blockValueReader.apply(firstValueIndex + i); + if (values.contains(value)) { + continue; // Skip duplicates + } + values.add(value); + builder.add(dateFieldType.equalityQuery(value, searchExecutionContext), BooleanClause.Occur.SHOULD); + } + yield new ConstantScoreQuery(builder.build()); + } + }; + } + } + private static class GeoShapeQueryList extends QueryList { private final BytesRef scratch = new BytesRef(); private final IntFunction blockValueReader; diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/CsvAssert.java b/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/CsvAssert.java index 4e31916d5328e..e409c18c69169 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/CsvAssert.java +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/CsvAssert.java @@ -278,7 +278,7 @@ private static void dataFailure( fail(description + System.lineSeparator() + describeFailures(dataFailures) + actual + expected); } - private static final int MAX_ROWS = 25; + private static final int MAX_ROWS = 50; private static String pipeTable( String description, diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/CsvTestsDataLoader.java b/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/CsvTestsDataLoader.java index d6c0773995233..eefcb3ab7581b 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/CsvTestsDataLoader.java +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/CsvTestsDataLoader.java @@ -81,6 +81,8 @@ public class CsvTestsDataLoader { private static final TestDataset SAMPLE_DATA_TS_NANOS = SAMPLE_DATA.withIndex("sample_data_ts_nanos") .withData("sample_data_ts_nanos.csv") .withTypeMapping(Map.of("@timestamp", "date_nanos")); + private static final TestDataset LOOKUP_SAMPLE_DATA_TS_NANOS = SAMPLE_DATA_TS_NANOS.withIndex("lookup_sample_data_ts_nanos") + .withSetting("lookup-settings.json"); private static final TestDataset MISSING_IP_SAMPLE_DATA = new TestDataset("missing_ip_sample_data"); private static final TestDataset CLIENT_IPS = new TestDataset("clientips"); private static final TestDataset CLIENT_IPS_LOOKUP = CLIENT_IPS.withIndex("clientips_lookup").withSetting("lookup-settings.json"); @@ -131,6 +133,7 @@ public class CsvTestsDataLoader { Map.entry(SAMPLE_DATA_STR.indexName, SAMPLE_DATA_STR), Map.entry(SAMPLE_DATA_TS_LONG.indexName, SAMPLE_DATA_TS_LONG), Map.entry(SAMPLE_DATA_TS_NANOS.indexName, SAMPLE_DATA_TS_NANOS), + Map.entry(LOOKUP_SAMPLE_DATA_TS_NANOS.indexName, LOOKUP_SAMPLE_DATA_TS_NANOS), Map.entry(MISSING_IP_SAMPLE_DATA.indexName, MISSING_IP_SAMPLE_DATA), Map.entry(CLIENT_IPS.indexName, CLIENT_IPS), Map.entry(CLIENT_IPS_LOOKUP.indexName, CLIENT_IPS_LOOKUP), diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/lookup-join.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/lookup-join.csv-spec index a3c97b524d10a..f415c0b415efc 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/lookup-join.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/lookup-join.csv-spec @@ -1544,3 +1544,27 @@ French Spanish German ; + +############################################### +# LOOKUP JOIN on date_nanos field +############################################### + +joinDateNanos +required_capability: join_lookup_v12 +required_capability: date_nanos_lookup_join + +FROM sample_data_ts_nanos +| LOOKUP JOIN lookup_sample_data_ts_nanos ON @timestamp +| KEEP @timestamp, client_ip, event_duration, message +| SORT @timestamp DESC +; + +@timestamp:date_nanos | client_ip:ip | event_duration:long | message:keyword +2023-10-23T13:55:01.543123456Z | 172.21.3.15 | 1756467 | Connected to 10.1.0.1 +2023-10-23T13:53:55.832123456Z | 172.21.3.15 | 5033755 | Connection error +2023-10-23T13:52:55.015123456Z | 172.21.3.15 | 8268153 | Connection error +2023-10-23T13:51:54.732123456Z | 172.21.3.15 | 725448 | Connection error +2023-10-23T13:33:34.937123456Z | 172.21.0.5 | 1232382 | Disconnected +2023-10-23T12:27:28.948123456Z | 172.21.2.113 | 2764889 | Connected to 10.1.0.2 +2023-10-23T12:15:03.360123456Z | 172.21.2.162 | 3450233 | Connected to 10.1.0.3 +; diff --git a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/LookupJoinTypesIT.java b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/LookupJoinTypesIT.java index 52c41e4056a8e..3107a07bce6fc 100644 --- a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/LookupJoinTypesIT.java +++ b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/LookupJoinTypesIT.java @@ -40,6 +40,7 @@ import static org.elasticsearch.xpack.esql.core.type.DataType.BOOLEAN; import static org.elasticsearch.xpack.esql.core.type.DataType.BYTE; import static org.elasticsearch.xpack.esql.core.type.DataType.DATETIME; +import static org.elasticsearch.xpack.esql.core.type.DataType.DATE_NANOS; import static org.elasticsearch.xpack.esql.core.type.DataType.DOC_DATA_TYPE; import static org.elasticsearch.xpack.esql.core.type.DataType.DOUBLE; import static org.elasticsearch.xpack.esql.core.type.DataType.FLOAT; @@ -174,6 +175,19 @@ protected Collection> nodePlugins() { } } + // Tests for mixed-date/time types + var dateTypes = List.of(DATETIME, DATE_NANOS); + { + TestConfigs configs = testConfigurations.computeIfAbsent("mixed-temporal", TestConfigs::new); + for (DataType mainType : dateTypes) { + for (DataType lookupType : dateTypes) { + if (mainType != lookupType) { + configs.addFails(mainType, lookupType); + } + } + } + } + // Tests for all unsupported types DataType[] unsupported = Join.UNSUPPORTED_TYPES; { @@ -199,7 +213,20 @@ protected Collection> nodePlugins() { } // Tests for all types where left and right are the same type - DataType[] supported = { BOOLEAN, LONG, INTEGER, DOUBLE, SHORT, BYTE, FLOAT, HALF_FLOAT, DATETIME, IP, KEYWORD, SCALED_FLOAT }; + DataType[] supported = { + BOOLEAN, + LONG, + INTEGER, + DOUBLE, + SHORT, + BYTE, + FLOAT, + HALF_FLOAT, + DATETIME, + DATE_NANOS, + IP, + KEYWORD, + SCALED_FLOAT }; { Collection existing = testConfigurations.values(); TestConfigs configs = testConfigurations.computeIfAbsent("same", TestConfigs::new); @@ -275,6 +302,10 @@ public void testLookupJoinMixedNumerical() { testLookupJoinTypes("mixed-numerical"); } + public void testLookupJoinMixedTemporal() { + testLookupJoinTypes("mixed-temporal"); + } + public void testLookupJoinSame() { testLookupJoinTypes("same"); } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java index 80509b7519bff..abe66e7308225 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java @@ -418,6 +418,12 @@ public enum Cap { * e.g. {@code WHERE millis > to_datenanos("2023-10-23T12:15:03.360103847") AND millis < to_datetime("2023-10-23T13:53:55.832")} */ FIX_DATE_NANOS_MIXED_RANGE_PUSHDOWN_BUG(), + + /** + * Support for date nanos in lookup join. Done in #127962 + */ + DATE_NANOS_LOOKUP_JOIN, + /** * DATE_PARSE supports reading timezones */ diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/enrich/AbstractLookupService.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/enrich/AbstractLookupService.java index b5caa42577311..fa6ab872e2d90 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/enrich/AbstractLookupService.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/enrich/AbstractLookupService.java @@ -201,6 +201,7 @@ protected static QueryList termQueryList( return switch (inputDataType) { case IP -> QueryList.ipTermQueryList(field, searchExecutionContext, (BytesRefBlock) block); case DATETIME -> QueryList.dateTermQueryList(field, searchExecutionContext, (LongBlock) block); + case DATE_NANOS -> QueryList.dateNanosTermQueryList(field, searchExecutionContext, (LongBlock) block); default -> QueryList.rawTermQueryList(field, searchExecutionContext, block); }; } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/join/Join.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/join/Join.java index af8a32d22ca99..e9fb082ec0b1c 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/join/Join.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/join/Join.java @@ -37,7 +37,6 @@ import static org.elasticsearch.xpack.esql.core.type.DataType.COUNTER_DOUBLE; import static org.elasticsearch.xpack.esql.core.type.DataType.COUNTER_INTEGER; import static org.elasticsearch.xpack.esql.core.type.DataType.COUNTER_LONG; -import static org.elasticsearch.xpack.esql.core.type.DataType.DATE_NANOS; import static org.elasticsearch.xpack.esql.core.type.DataType.DATE_PERIOD; import static org.elasticsearch.xpack.esql.core.type.DataType.DOC_DATA_TYPE; import static org.elasticsearch.xpack.esql.core.type.DataType.GEO_POINT; @@ -70,7 +69,6 @@ public class Join extends BinaryPlan implements PostAnalysisVerificationAware, S COUNTER_LONG, COUNTER_INTEGER, COUNTER_DOUBLE, - DATE_NANOS, OBJECT, SOURCE, DATE_PERIOD, From c2b3b5dbf78d404a92e8ea53d1e751f6490240aa Mon Sep 17 00:00:00 2001 From: Craig Taverner Date: Mon, 9 Jun 2025 11:55:11 +0200 Subject: [PATCH 4/9] Fix compile error after backport --- .../index/mapper/DateFieldMapper.java | 15 ++------------- 1 file changed, 2 insertions(+), 13 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java index 55d29d37ad478..a102f6c8e55ad 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java @@ -474,18 +474,7 @@ public DateFieldType(String name) { } public DateFieldType(String name, boolean isIndexed, Resolution resolution) { - this( - name, - isIndexed, - isIndexed, - false, - true, - DEFAULT_DATE_TIME_FORMATTER, - resolution, - null, - null, - Collections.emptyMap() - ); + this(name, isIndexed, isIndexed, false, true, DEFAULT_DATE_TIME_FORMATTER, resolution, null, null, Collections.emptyMap()); } public DateFieldType(String name, boolean isIndexed) { @@ -746,7 +735,7 @@ public Query rangeQuery( query = SortedNumericDocValuesField.newSlowRangeQuery(name(), l, u); } if (hasDocValues() && context.indexSortedOnField(name())) { - query = new XIndexSortSortedNumericDocValuesRangeQuery(name(), l, u, query); + query = new IndexSortSortedNumericDocValuesRangeQuery(name(), l, u, query); } return query; } From e2a92b3003a1d2516024f9fb448a912ec14a2ba4 Mon Sep 17 00:00:00 2001 From: Craig Taverner Date: Mon, 9 Jun 2025 11:59:59 +0200 Subject: [PATCH 5/9] Fix compile error after backport --- .../org/elasticsearch/xpack/esql/action/LookupJoinTypesIT.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/LookupJoinTypesIT.java b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/LookupJoinTypesIT.java index 3107a07bce6fc..e8ee13cf569de 100644 --- a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/LookupJoinTypesIT.java +++ b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/LookupJoinTypesIT.java @@ -527,7 +527,7 @@ private static String propertySpecFor(String fieldName, DataType type, String ex private static void validateIndex(String indexName, String fieldName, Object expectedValue) { String query = String.format(Locale.ROOT, "FROM %s | KEEP %s", indexName, fieldName); try (var response = EsqlQueryRequestBuilder.newRequestBuilder(client()).query(query).get()) { - ColumnInfo info = response.response().columns().getFirst(); + ColumnInfo info = response.response().columns().get(0); assertThat("Expected index '" + indexName + "' to have column '" + fieldName + ": " + query, info.name(), is(fieldName)); Iterator results = response.response().column(0).iterator(); assertTrue("Expected at least one result for query: " + query, results.hasNext()); From bb726e7e505c58050fe0c02071f60217747f31ea Mon Sep 17 00:00:00 2001 From: Craig Taverner Date: Mon, 9 Jun 2025 12:06:23 +0200 Subject: [PATCH 6/9] Update docs/changelog/129138.yaml --- docs/changelog/129138.yaml | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 docs/changelog/129138.yaml diff --git a/docs/changelog/129138.yaml b/docs/changelog/129138.yaml new file mode 100644 index 0000000000000..86144dcaff27c --- /dev/null +++ b/docs/changelog/129138.yaml @@ -0,0 +1,5 @@ +pr: 129138 +summary: "[8.18] backport LOOKUP JOIN type tests and DATE_NANOS" +area: ES|QL +type: bug +issues: [] From 029ab7c697890f8b392a8d04b235e04a502bb83e Mon Sep 17 00:00:00 2001 From: Craig Taverner Date: Mon, 9 Jun 2025 14:43:13 +0200 Subject: [PATCH 7/9] SEMANTIC_TEXT was removed in later PRs, so not really testable in 8.18. --- .../org/elasticsearch/xpack/esql/action/LookupJoinTypesIT.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/LookupJoinTypesIT.java b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/LookupJoinTypesIT.java index e8ee13cf569de..a8c90988bd461 100644 --- a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/LookupJoinTypesIT.java +++ b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/LookupJoinTypesIT.java @@ -51,6 +51,7 @@ import static org.elasticsearch.xpack.esql.core.type.DataType.LONG; import static org.elasticsearch.xpack.esql.core.type.DataType.NULL; import static org.elasticsearch.xpack.esql.core.type.DataType.SCALED_FLOAT; +import static org.elasticsearch.xpack.esql.core.type.DataType.SEMANTIC_TEXT; import static org.elasticsearch.xpack.esql.core.type.DataType.SHORT; import static org.elasticsearch.xpack.esql.core.type.DataType.TEXT; import static org.elasticsearch.xpack.esql.core.type.DataType.TSID_DATA_TYPE; @@ -249,7 +250,7 @@ protected Collection> nodePlugins() { for (DataType type : DataType.values()) { boolean isUnsupported = List.of(unsupported).contains(type); boolean isSupported = List.of(supported).contains(type); - if (isUnsupported == false && isSupported == false) { + if (isUnsupported == false && isSupported == false && type != SEMANTIC_TEXT) { missing.add(type); } } From d1a60bd91cb48ea0f4b67b580429d1f822a91356 Mon Sep 17 00:00:00 2001 From: Craig Taverner Date: Mon, 9 Jun 2025 14:44:55 +0200 Subject: [PATCH 8/9] Delete docs/changelog/129138.yaml --- docs/changelog/129138.yaml | 5 ----- 1 file changed, 5 deletions(-) delete mode 100644 docs/changelog/129138.yaml diff --git a/docs/changelog/129138.yaml b/docs/changelog/129138.yaml deleted file mode 100644 index 86144dcaff27c..0000000000000 --- a/docs/changelog/129138.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 129138 -summary: "[8.18] backport LOOKUP JOIN type tests and DATE_NANOS" -area: ES|QL -type: bug -issues: [] From a3f0dfe6012e3703d87ff806cd08fbfc2c621199 Mon Sep 17 00:00:00 2001 From: Craig Taverner Date: Mon, 9 Jun 2025 14:44:55 +0200 Subject: [PATCH 9/9] Removed incorrectly added changelog This is a backport --- docs/changelog/129138.yaml | 5 ----- 1 file changed, 5 deletions(-) delete mode 100644 docs/changelog/129138.yaml diff --git a/docs/changelog/129138.yaml b/docs/changelog/129138.yaml deleted file mode 100644 index 86144dcaff27c..0000000000000 --- a/docs/changelog/129138.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 129138 -summary: "[8.18] backport LOOKUP JOIN type tests and DATE_NANOS" -area: ES|QL -type: bug -issues: []