diff --git a/docs/changelog/132162.yaml b/docs/changelog/132162.yaml new file mode 100644 index 0000000000000..2622ac702ce9a --- /dev/null +++ b/docs/changelog/132162.yaml @@ -0,0 +1,6 @@ +pr: 132162 +summary: Fix default missing index sort value of `data_nanos` pre 7.14 +area: Search +type: bug +issues: + - 132040 diff --git a/server/src/internalClusterTest/java/org/elasticsearch/index/IndexSortIT.java b/server/src/internalClusterTest/java/org/elasticsearch/index/IndexSortIT.java index 0e85d2b9750cb..d76b7d0dd250c 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/index/IndexSortIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/index/IndexSortIT.java @@ -11,6 +11,7 @@ import org.apache.lucene.search.Sort; import org.apache.lucene.search.SortField; +import org.apache.lucene.search.SortedNumericSelector; import org.apache.lucene.search.SortedNumericSortField; import org.apache.lucene.search.SortedSetSortField; import org.elasticsearch.common.settings.Settings; @@ -80,6 +81,33 @@ public void testIndexSort() { assertSortedSegments("test", indexSort); } + public void testIndexSortDateNanos() { + prepareCreate("test").setSettings( + Settings.builder() + .put(indexSettings()) + .put("index.number_of_shards", "1") + .put("index.number_of_replicas", "1") + .put("index.sort.field", "@timestamp") + .put("index.sort.order", "desc") + ).setMapping(""" + { + "properties": { + "@timestamp": { + "type": "date_nanos" + } + } + } + """).get(); + + flushAndRefresh(); + ensureYellow(); + + SortField sf = new SortedNumericSortField("@timestamp", SortField.Type.LONG, true, SortedNumericSelector.Type.MAX); + sf.setMissingValue(0L); + Sort expectedIndexSort = new Sort(sf); + assertSortedSegments("test", expectedIndexSort); + } + public void testInvalidIndexSort() { IllegalArgumentException exc = expectThrows( IllegalArgumentException.class, diff --git a/server/src/main/java/org/elasticsearch/index/fielddata/IndexNumericFieldData.java b/server/src/main/java/org/elasticsearch/index/fielddata/IndexNumericFieldData.java index 9956f8105b079..86d962f55a558 100644 --- a/server/src/main/java/org/elasticsearch/index/fielddata/IndexNumericFieldData.java +++ b/server/src/main/java/org/elasticsearch/index/fielddata/IndexNumericFieldData.java @@ -146,12 +146,21 @@ public SortField sortField( boolean reverse ) { SortField sortField = sortField(missingValue, sortMode, nested, reverse); - // we introduced INT sort type in 8.19 and from 9.1 - if (getNumericType().sortFieldType != SortField.Type.INT + if (getNumericType() == NumericType.DATE_NANOSECONDS + && indexCreatedVersion.before(IndexVersions.V_7_14_0) + && missingValue == null + && Long.valueOf(0L).equals(sortField.getMissingValue())) { + // 7.14 changed the default missing value of sort on date_nanos, from Long.MIN_VALUE + // to 0L - for compatibility we require to a missing value of MIN_VALUE to allow to + // open the index. + sortField.setMissingValue(Long.MIN_VALUE); + return sortField; + } else if (getNumericType().sortFieldType != SortField.Type.INT + // we introduced INT sort type in 8.19 and from 9.1 || indexCreatedVersion.onOrAfter(IndexVersions.INDEX_INT_SORT_INT_TYPE) || indexCreatedVersion.between(IndexVersions.INDEX_INT_SORT_INT_TYPE_8_19, UPGRADE_TO_LUCENE_10_0_0)) { - return sortField; - } + return sortField; + } if ((sortField instanceof SortedNumericSortField) == false) { return sortField; } diff --git a/server/src/test/java/org/elasticsearch/index/IndexSortSettingsTests.java b/server/src/test/java/org/elasticsearch/index/IndexSortSettingsTests.java index 7221d69b74d46..52cc1348fd48a 100644 --- a/server/src/test/java/org/elasticsearch/index/IndexSortSettingsTests.java +++ b/server/src/test/java/org/elasticsearch/index/IndexSortSettingsTests.java @@ -13,6 +13,7 @@ import org.apache.lucene.search.Sort; import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.time.DateUtils; import org.elasticsearch.common.util.Maps; import org.elasticsearch.index.fielddata.FieldDataContext; import org.elasticsearch.index.fielddata.IndexFieldData; @@ -30,9 +31,12 @@ import org.elasticsearch.search.sort.SortOrder; import org.elasticsearch.test.ESTestCase; +import java.util.ArrayList; import java.util.Collections; +import java.util.List; import java.util.Map; import java.util.Set; +import java.util.stream.Stream; import static org.elasticsearch.index.IndexSettingsTests.newIndexMeta; import static org.hamcrest.Matchers.arrayWithSize; @@ -174,6 +178,58 @@ public void testSortingAgainstAliasesPre713() { ); } + public void testSortMissingValueDateNanoFieldPre714() { + MappedFieldType tsField = new DateFieldMapper.DateFieldType("@timestamp", true, DateFieldMapper.Resolution.NANOSECONDS); + var indexSettingsBuilder = Settings.builder(); + indexSettingsBuilder.put("index.sort.field", "@timestamp"); + indexSettingsBuilder.put("index.sort.order", "desc"); + + // test with index version 7.13 and before + var pre714Versions = Stream.concat(Stream.of(IndexVersions.V_7_13_0), randomVersionsBefore(IndexVersions.V_7_13_0)).toList(); + for (var version : pre714Versions) { + indexSettingsBuilder.put(IndexMetadata.SETTING_VERSION_CREATED, version); + Sort sort = buildIndexSort(indexSettings(indexSettingsBuilder.build()), Map.of("@timestamp", tsField)); + assertThat(sort.getSort(), arrayWithSize(1)); + assertThat(sort.getSort()[0].getField(), equalTo("@timestamp")); + assertThat(sort.getSort()[0].getMissingValue(), equalTo(Long.MIN_VALUE)); + } + + // now test with index version 7.14 and after + var post713Versions = Stream.concat(Stream.of(IndexVersions.V_7_14_0), randomVersionsAfter(IndexVersions.V_7_14_0)).toList(); + for (var version : post713Versions) { + indexSettingsBuilder.put(IndexMetadata.SETTING_VERSION_CREATED, version); + Sort sort = buildIndexSort(indexSettings(indexSettingsBuilder.build()), Map.of("@timestamp", tsField)); + assertThat(sort.getSort(), arrayWithSize(1)); + assertThat(sort.getSort()[0].getField(), equalTo("@timestamp")); + assertThat(sort.getSort()[0].getMissingValue(), equalTo(0L)); + } + + // asc order has not changed behaviour in any version + indexSettingsBuilder.put("index.sort.order", "asc"); + var allVersions = Stream.concat(post713Versions.stream(), pre714Versions.stream()).toList(); + for (var version : allVersions) { + indexSettingsBuilder.put(IndexMetadata.SETTING_VERSION_CREATED, version); + Sort sort = buildIndexSort(indexSettings(indexSettingsBuilder.build()), Map.of("@timestamp", tsField)); + assertThat(sort.getSort(), arrayWithSize(1)); + assertThat(sort.getSort()[0].getField(), equalTo("@timestamp")); + assertThat(sort.getSort()[0].getMissingValue(), equalTo(DateUtils.MAX_NANOSECOND)); + } + + // ensure no change in behaviour when a missing value is set + indexSettingsBuilder.put("index.sort.missing", "_first"); + for (var version : allVersions) { + indexSettingsBuilder.put(IndexMetadata.SETTING_VERSION_CREATED, version); + Sort sort = buildIndexSort(indexSettings(indexSettingsBuilder.build()), Map.of("@timestamp", tsField)); + assertThat(sort.getSort()[0].getMissingValue(), equalTo(0L)); + } + indexSettingsBuilder.put("index.sort.missing", "_last"); + for (var version : allVersions) { + indexSettingsBuilder.put(IndexMetadata.SETTING_VERSION_CREATED, version); + Sort sort = buildIndexSort(indexSettings(indexSettingsBuilder.build()), Map.of("@timestamp", tsField)); + assertThat(sort.getSort()[0].getMissingValue(), equalTo(Long.MAX_VALUE)); + } + } + public void testTimeSeriesMode() { IndexSettings indexSettings = indexSettings( Settings.builder() @@ -224,4 +280,24 @@ private Sort buildIndexSort(IndexSettings indexSettings, Map randomVersionsBefore(IndexVersion indexVersion) { + var versions = IndexVersions.getAllVersions().stream().filter(v -> v.before(indexVersion)).toList(); + List ret = new ArrayList<>(); + for (int i = 0; i < 10; i++) { + ret.add(randomValueOtherThanMany(ret::contains, () -> randomFrom(versions))); + } + return ret.stream(); + } + + /* Returns a stream of versions after the given version */ + Stream randomVersionsAfter(IndexVersion indexVersion) { + var versions = IndexVersions.getAllVersions().stream().filter(v -> v.after(indexVersion)).toList(); + List ret = new ArrayList<>(); + for (int i = 0; i < 10; i++) { + ret.add(randomValueOtherThanMany(ret::contains, () -> randomFrom(versions))); + } + return ret.stream(); + } }