From e3a4839482dea0f96e2adb7e3ffc0fe888da28e1 Mon Sep 17 00:00:00 2001 From: ChrisHegarty Date: Wed, 30 Jul 2025 08:54:33 +0100 Subject: [PATCH 1/3] Fix default missing index sort value of data_nanos pre 7.14 --- .../org/elasticsearch/index/IndexSortIT.java | 28 +++++++ .../fielddata/IndexNumericFieldData.java | 17 ++++- .../index/IndexSortSettingsTests.java | 76 +++++++++++++++++++ 3 files changed, 117 insertions(+), 4 deletions(-) diff --git a/server/src/internalClusterTest/java/org/elasticsearch/index/IndexSortIT.java b/server/src/internalClusterTest/java/org/elasticsearch/index/IndexSortIT.java index 0e85d2b9750cb..d76b7d0dd250c 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/index/IndexSortIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/index/IndexSortIT.java @@ -11,6 +11,7 @@ import org.apache.lucene.search.Sort; import org.apache.lucene.search.SortField; +import org.apache.lucene.search.SortedNumericSelector; import org.apache.lucene.search.SortedNumericSortField; import org.apache.lucene.search.SortedSetSortField; import org.elasticsearch.common.settings.Settings; @@ -80,6 +81,33 @@ public void testIndexSort() { assertSortedSegments("test", indexSort); } + public void testIndexSortDateNanos() { + prepareCreate("test").setSettings( + Settings.builder() + .put(indexSettings()) + .put("index.number_of_shards", "1") + .put("index.number_of_replicas", "1") + .put("index.sort.field", "@timestamp") + .put("index.sort.order", "desc") + ).setMapping(""" + { + "properties": { + "@timestamp": { + "type": "date_nanos" + } + } + } + """).get(); + + flushAndRefresh(); + ensureYellow(); + + SortField sf = new SortedNumericSortField("@timestamp", SortField.Type.LONG, true, SortedNumericSelector.Type.MAX); + sf.setMissingValue(0L); + Sort expectedIndexSort = new Sort(sf); + assertSortedSegments("test", expectedIndexSort); + } + public void testInvalidIndexSort() { IllegalArgumentException exc = expectThrows( IllegalArgumentException.class, diff --git a/server/src/main/java/org/elasticsearch/index/fielddata/IndexNumericFieldData.java b/server/src/main/java/org/elasticsearch/index/fielddata/IndexNumericFieldData.java index 9956f8105b079..86d962f55a558 100644 --- a/server/src/main/java/org/elasticsearch/index/fielddata/IndexNumericFieldData.java +++ b/server/src/main/java/org/elasticsearch/index/fielddata/IndexNumericFieldData.java @@ -146,12 +146,21 @@ public SortField sortField( boolean reverse ) { SortField sortField = sortField(missingValue, sortMode, nested, reverse); - // we introduced INT sort type in 8.19 and from 9.1 - if (getNumericType().sortFieldType != SortField.Type.INT + if (getNumericType() == NumericType.DATE_NANOSECONDS + && indexCreatedVersion.before(IndexVersions.V_7_14_0) + && missingValue == null + && Long.valueOf(0L).equals(sortField.getMissingValue())) { + // 7.14 changed the default missing value of sort on date_nanos, from Long.MIN_VALUE + // to 0L - for compatibility we require to a missing value of MIN_VALUE to allow to + // open the index. + sortField.setMissingValue(Long.MIN_VALUE); + return sortField; + } else if (getNumericType().sortFieldType != SortField.Type.INT + // we introduced INT sort type in 8.19 and from 9.1 || indexCreatedVersion.onOrAfter(IndexVersions.INDEX_INT_SORT_INT_TYPE) || indexCreatedVersion.between(IndexVersions.INDEX_INT_SORT_INT_TYPE_8_19, UPGRADE_TO_LUCENE_10_0_0)) { - return sortField; - } + return sortField; + } if ((sortField instanceof SortedNumericSortField) == false) { return sortField; } diff --git a/server/src/test/java/org/elasticsearch/index/IndexSortSettingsTests.java b/server/src/test/java/org/elasticsearch/index/IndexSortSettingsTests.java index 7221d69b74d46..420e8e41fd9e7 100644 --- a/server/src/test/java/org/elasticsearch/index/IndexSortSettingsTests.java +++ b/server/src/test/java/org/elasticsearch/index/IndexSortSettingsTests.java @@ -13,6 +13,7 @@ import org.apache.lucene.search.Sort; import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.time.DateUtils; import org.elasticsearch.common.util.Maps; import org.elasticsearch.index.fielddata.FieldDataContext; import org.elasticsearch.index.fielddata.IndexFieldData; @@ -30,9 +31,12 @@ import org.elasticsearch.search.sort.SortOrder; import org.elasticsearch.test.ESTestCase; +import java.util.ArrayList; import java.util.Collections; +import java.util.List; import java.util.Map; import java.util.Set; +import java.util.stream.Stream; import static org.elasticsearch.index.IndexSettingsTests.newIndexMeta; import static org.hamcrest.Matchers.arrayWithSize; @@ -174,6 +178,58 @@ public void testSortingAgainstAliasesPre713() { ); } + public void testSortMissingValueDateNanoFieldPre714() { + MappedFieldType tsField = new DateFieldMapper.DateFieldType("@timestamp", true, DateFieldMapper.Resolution.NANOSECONDS); + var indexSettingsBuilder = Settings.builder(); + indexSettingsBuilder.put("index.sort.field", "@timestamp"); + indexSettingsBuilder.put("index.sort.order", "desc"); + + // test with index version 7.13 and before + var pre714Versions = Stream.concat(Stream.of(IndexVersions.V_7_13_0), randomVersionsBefore(IndexVersions.V_7_13_0)).toList(); + for (var version : pre714Versions) { + indexSettingsBuilder.put(IndexMetadata.SETTING_VERSION_CREATED, version); + Sort sort = buildIndexSort(indexSettings(indexSettingsBuilder.build()), Map.of("@timestamp", tsField)); + assertThat(sort.getSort(), arrayWithSize(1)); + assertThat(sort.getSort()[0].getField(), equalTo("@timestamp")); + assertThat(sort.getSort()[0].getMissingValue(), equalTo(Long.MIN_VALUE)); + } + + // now test with index version 7.14 and after + var post713Versions = Stream.concat(Stream.of(IndexVersions.V_7_14_0), randomVersionsAfter(IndexVersions.V_7_14_0)).toList(); + for (var version : post713Versions) { + indexSettingsBuilder.put(IndexMetadata.SETTING_VERSION_CREATED, version); + Sort sort = buildIndexSort(indexSettings(indexSettingsBuilder.build()), Map.of("@timestamp", tsField)); + assertThat(sort.getSort(), arrayWithSize(1)); + assertThat(sort.getSort()[0].getField(), equalTo("@timestamp")); + assertThat(sort.getSort()[0].getMissingValue(), equalTo(0L)); + } + + // asc order has not changed behaviour in any version + indexSettingsBuilder.put("index.sort.order", "asc"); + var allVersions = Stream.concat(post713Versions.stream(), pre714Versions.stream()).toList(); + for (var version : allVersions) { + indexSettingsBuilder.put(IndexMetadata.SETTING_VERSION_CREATED, version); + Sort sort = buildIndexSort(indexSettings(indexSettingsBuilder.build()), Map.of("@timestamp", tsField)); + assertThat(sort.getSort(), arrayWithSize(1)); + assertThat(sort.getSort()[0].getField(), equalTo("@timestamp")); + assertThat(sort.getSort()[0].getMissingValue(), equalTo(DateUtils.MAX_NANOSECOND)); + } + + // ensure when no change in behaviour when a missing value is set + indexSettingsBuilder.put("index.sort.missing", "_first"); + for (var version : allVersions) { + indexSettingsBuilder.put(IndexMetadata.SETTING_VERSION_CREATED, version); + Sort sort = buildIndexSort(indexSettings(indexSettingsBuilder.build()), Map.of("@timestamp", tsField)); + assertThat(sort.getSort()[0].getMissingValue(), equalTo(0L)); + } + indexSettingsBuilder.put("index.sort.missing", "_last"); + for (var version : allVersions) { + indexSettingsBuilder.put(IndexMetadata.SETTING_VERSION_CREATED, version); + Sort sort = buildIndexSort(indexSettings(indexSettingsBuilder.build()), Map.of("@timestamp", tsField)); + assertThat(sort.getSort()[0].getMissingValue(), equalTo(Long.MAX_VALUE)); + } + } + public void testTimeSeriesMode() { IndexSettings indexSettings = indexSettings( Settings.builder() @@ -224,4 +280,24 @@ private Sort buildIndexSort(IndexSettings indexSettings, Map randomVersionsBefore(IndexVersion indexVersion) { + var versions = IndexVersions.getAllVersions().stream().filter(v -> v.before(indexVersion)).toList(); + List ret = new ArrayList<>(); + for (int i = 0; i < 10; i++) { + ret.add(randomValueOtherThanMany(ret::contains, () -> randomFrom(versions))); + } + return ret.stream(); + } + + /* Returns a stream of versions after the given version */ + Stream randomVersionsAfter(IndexVersion indexVersion) { + var versions = IndexVersions.getAllVersions().stream().filter(v -> v.after(indexVersion)).toList(); + List ret = new ArrayList<>(); + for (int i = 0; i < 10; i++) { + ret.add(randomValueOtherThanMany(ret::contains, () -> randomFrom(versions))); + } + return ret.stream(); + } } From 69fd9efe62f9690bc6ab47ab448d6ba74087dfc5 Mon Sep 17 00:00:00 2001 From: Chris Hegarty <62058229+ChrisHegarty@users.noreply.github.com> Date: Wed, 30 Jul 2025 09:43:06 +0100 Subject: [PATCH 2/3] Update docs/changelog/132162.yaml --- docs/changelog/132162.yaml | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 docs/changelog/132162.yaml diff --git a/docs/changelog/132162.yaml b/docs/changelog/132162.yaml new file mode 100644 index 0000000000000..2622ac702ce9a --- /dev/null +++ b/docs/changelog/132162.yaml @@ -0,0 +1,6 @@ +pr: 132162 +summary: Fix default missing index sort value of `data_nanos` pre 7.14 +area: Search +type: bug +issues: + - 132040 From 541ba8d5d5a1fa6bff6ec57edaf35255e59e11de Mon Sep 17 00:00:00 2001 From: ChrisHegarty Date: Wed, 30 Jul 2025 09:49:02 +0100 Subject: [PATCH 3/3] itr --- .../java/org/elasticsearch/index/IndexSortSettingsTests.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/test/java/org/elasticsearch/index/IndexSortSettingsTests.java b/server/src/test/java/org/elasticsearch/index/IndexSortSettingsTests.java index 420e8e41fd9e7..52cc1348fd48a 100644 --- a/server/src/test/java/org/elasticsearch/index/IndexSortSettingsTests.java +++ b/server/src/test/java/org/elasticsearch/index/IndexSortSettingsTests.java @@ -215,7 +215,7 @@ public void testSortMissingValueDateNanoFieldPre714() { assertThat(sort.getSort()[0].getMissingValue(), equalTo(DateUtils.MAX_NANOSECOND)); } - // ensure when no change in behaviour when a missing value is set + // ensure no change in behaviour when a missing value is set indexSettingsBuilder.put("index.sort.missing", "_first"); for (var version : allVersions) { indexSettingsBuilder.put(IndexMetadata.SETTING_VERSION_CREATED, version);