Skip to content

Commit 3bcd9a1

Browse files
ChrisHegartyChrisHegarty
authored
[9.1.1] Fix default missing index sort value of data_nanos pre 7.14 (#132171)
This commit fixes an issue whereby indices created prior to 7.14 that have an index sort on a date_nanos field can no longer be opened (with versions >= 7.14). When opening an index the configured index sort, derived from the index settings, must match exactly that of the sort encoded in the index itself. A change to fix a bug back in 7.14 changed this for date_nanos fields whose index.sort.missing value is absent, see #74760. Specifically, the default minimum value changed from Long.MIN_VALUE to 0L. The change in this commit restores the default minimum value for indices prior to 7.14.
1 parent ec5785f commit 3bcd9a1

File tree

4 files changed

+123
-4
lines changed

4 files changed

+123
-4
lines changed

‎docs/changelog/132162.yaml

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
pr: 132162
2+
summary: Fix default missing index sort value of `data_nanos` pre 7.14
3+
area: Search
4+
type: bug
5+
issues:
6+
- 132040

‎server/src/internalClusterTest/java/org/elasticsearch/index/IndexSortIT.java

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@
1111

1212
import org.apache.lucene.search.Sort;
1313
import org.apache.lucene.search.SortField;
14+
import org.apache.lucene.search.SortedNumericSelector;
1415
import org.apache.lucene.search.SortedNumericSortField;
1516
import org.apache.lucene.search.SortedSetSortField;
1617
import org.elasticsearch.common.settings.Settings;
@@ -80,6 +81,33 @@ public void testIndexSort() {
8081
assertSortedSegments("test", indexSort);
8182
}
8283

84+
public void testIndexSortDateNanos() {
85+
prepareCreate("test").setSettings(
86+
Settings.builder()
87+
.put(indexSettings())
88+
.put("index.number_of_shards", "1")
89+
.put("index.number_of_replicas", "1")
90+
.put("index.sort.field", "@timestamp")
91+
.put("index.sort.order", "desc")
92+
).setMapping("""
93+
{
94+
"properties": {
95+
"@timestamp": {
96+
"type": "date_nanos"
97+
}
98+
}
99+
}
100+
""").get();
101+
102+
flushAndRefresh();
103+
ensureYellow();
104+
105+
SortField sf = new SortedNumericSortField("@timestamp", SortField.Type.LONG, true, SortedNumericSelector.Type.MAX);
106+
sf.setMissingValue(0L);
107+
Sort expectedIndexSort = new Sort(sf);
108+
assertSortedSegments("test", expectedIndexSort);
109+
}
110+
83111
public void testInvalidIndexSort() {
84112
IllegalArgumentException exc = expectThrows(
85113
IllegalArgumentException.class,

‎server/src/main/java/org/elasticsearch/index/fielddata/IndexNumericFieldData.java

Lines changed: 13 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -146,12 +146,21 @@ public SortField sortField(
146146
boolean reverse
147147
) {
148148
SortField sortField = sortField(missingValue, sortMode, nested, reverse);
149-
// we introduced INT sort type in 8.19 and from 9.1
150-
if (getNumericType().sortFieldType != SortField.Type.INT
149+
if (getNumericType() == NumericType.DATE_NANOSECONDS
150+
&& indexCreatedVersion.before(IndexVersions.V_7_14_0)
151+
&& missingValue == null
152+
&& Long.valueOf(0L).equals(sortField.getMissingValue())) {
153+
// 7.14 changed the default missing value of sort on date_nanos, from Long.MIN_VALUE
154+
// to 0L - for compatibility we require to a missing value of MIN_VALUE to allow to
155+
// open the index.
156+
sortField.setMissingValue(Long.MIN_VALUE);
157+
return sortField;
158+
} else if (getNumericType().sortFieldType != SortField.Type.INT
159+
// we introduced INT sort type in 8.19 and from 9.1
151160
|| indexCreatedVersion.onOrAfter(IndexVersions.INDEX_INT_SORT_INT_TYPE)
152161
|| indexCreatedVersion.between(IndexVersions.INDEX_INT_SORT_INT_TYPE_8_19, UPGRADE_TO_LUCENE_10_0_0)) {
153-
return sortField;
154-
}
162+
return sortField;
163+
}
155164
if ((sortField instanceof SortedNumericSortField) == false) {
156165
return sortField;
157166
}

‎server/src/test/java/org/elasticsearch/index/IndexSortSettingsTests.java

Lines changed: 76 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@
1313
import org.apache.lucene.search.Sort;
1414
import org.elasticsearch.cluster.metadata.IndexMetadata;
1515
import org.elasticsearch.common.settings.Settings;
16+
import org.elasticsearch.common.time.DateUtils;
1617
import org.elasticsearch.common.util.Maps;
1718
import org.elasticsearch.index.fielddata.FieldDataContext;
1819
import org.elasticsearch.index.fielddata.IndexFieldData;
@@ -30,9 +31,12 @@
3031
import org.elasticsearch.search.sort.SortOrder;
3132
import org.elasticsearch.test.ESTestCase;
3233

34+
import java.util.ArrayList;
3335
import java.util.Collections;
36+
import java.util.List;
3437
import java.util.Map;
3538
import java.util.Set;
39+
import java.util.stream.Stream;
3640

3741
import static org.elasticsearch.index.IndexSettingsTests.newIndexMeta;
3842
import static org.hamcrest.Matchers.arrayWithSize;
@@ -174,6 +178,58 @@ public void testSortingAgainstAliasesPre713() {
174178
);
175179
}
176180

181+
public void testSortMissingValueDateNanoFieldPre714() {
182+
MappedFieldType tsField = new DateFieldMapper.DateFieldType("@timestamp", true, DateFieldMapper.Resolution.NANOSECONDS);
183+
var indexSettingsBuilder = Settings.builder();
184+
indexSettingsBuilder.put("index.sort.field", "@timestamp");
185+
indexSettingsBuilder.put("index.sort.order", "desc");
186+
187+
// test with index version 7.13 and before
188+
var pre714Versions = Stream.concat(Stream.of(IndexVersions.V_7_13_0), randomVersionsBefore(IndexVersions.V_7_13_0)).toList();
189+
for (var version : pre714Versions) {
190+
indexSettingsBuilder.put(IndexMetadata.SETTING_VERSION_CREATED, version);
191+
Sort sort = buildIndexSort(indexSettings(indexSettingsBuilder.build()), Map.of("@timestamp", tsField));
192+
assertThat(sort.getSort(), arrayWithSize(1));
193+
assertThat(sort.getSort()[0].getField(), equalTo("@timestamp"));
194+
assertThat(sort.getSort()[0].getMissingValue(), equalTo(Long.MIN_VALUE));
195+
}
196+
197+
// now test with index version 7.14 and after
198+
var post713Versions = Stream.concat(Stream.of(IndexVersions.V_7_14_0), randomVersionsAfter(IndexVersions.V_7_14_0)).toList();
199+
for (var version : post713Versions) {
200+
indexSettingsBuilder.put(IndexMetadata.SETTING_VERSION_CREATED, version);
201+
Sort sort = buildIndexSort(indexSettings(indexSettingsBuilder.build()), Map.of("@timestamp", tsField));
202+
assertThat(sort.getSort(), arrayWithSize(1));
203+
assertThat(sort.getSort()[0].getField(), equalTo("@timestamp"));
204+
assertThat(sort.getSort()[0].getMissingValue(), equalTo(0L));
205+
}
206+
207+
// asc order has not changed behaviour in any version
208+
indexSettingsBuilder.put("index.sort.order", "asc");
209+
var allVersions = Stream.concat(post713Versions.stream(), pre714Versions.stream()).toList();
210+
for (var version : allVersions) {
211+
indexSettingsBuilder.put(IndexMetadata.SETTING_VERSION_CREATED, version);
212+
Sort sort = buildIndexSort(indexSettings(indexSettingsBuilder.build()), Map.of("@timestamp", tsField));
213+
assertThat(sort.getSort(), arrayWithSize(1));
214+
assertThat(sort.getSort()[0].getField(), equalTo("@timestamp"));
215+
assertThat(sort.getSort()[0].getMissingValue(), equalTo(DateUtils.MAX_NANOSECOND));
216+
}
217+
218+
// ensure no change in behaviour when a missing value is set
219+
indexSettingsBuilder.put("index.sort.missing", "_first");
220+
for (var version : allVersions) {
221+
indexSettingsBuilder.put(IndexMetadata.SETTING_VERSION_CREATED, version);
222+
Sort sort = buildIndexSort(indexSettings(indexSettingsBuilder.build()), Map.of("@timestamp", tsField));
223+
assertThat(sort.getSort()[0].getMissingValue(), equalTo(0L));
224+
}
225+
indexSettingsBuilder.put("index.sort.missing", "_last");
226+
for (var version : allVersions) {
227+
indexSettingsBuilder.put(IndexMetadata.SETTING_VERSION_CREATED, version);
228+
Sort sort = buildIndexSort(indexSettings(indexSettingsBuilder.build()), Map.of("@timestamp", tsField));
229+
assertThat(sort.getSort()[0].getMissingValue(), equalTo(Long.MAX_VALUE));
230+
}
231+
}
232+
177233
public void testTimeSeriesMode() {
178234
IndexSettings indexSettings = indexSettings(
179235
Settings.builder()
@@ -224,4 +280,24 @@ private Sort buildIndexSort(IndexSettings indexSettings, Map<String, MappedField
224280
)
225281
);
226282
}
283+
284+
/* Returns a stream of versions before the given version */
285+
Stream<IndexVersion> randomVersionsBefore(IndexVersion indexVersion) {
286+
var versions = IndexVersions.getAllVersions().stream().filter(v -> v.before(indexVersion)).toList();
287+
List<IndexVersion> ret = new ArrayList<>();
288+
for (int i = 0; i < 10; i++) {
289+
ret.add(randomValueOtherThanMany(ret::contains, () -> randomFrom(versions)));
290+
}
291+
return ret.stream();
292+
}
293+
294+
/* Returns a stream of versions after the given version */
295+
Stream<IndexVersion> randomVersionsAfter(IndexVersion indexVersion) {
296+
var versions = IndexVersions.getAllVersions().stream().filter(v -> v.after(indexVersion)).toList();
297+
List<IndexVersion> ret = new ArrayList<>();
298+
for (int i = 0; i < 10; i++) {
299+
ret.add(randomValueOtherThanMany(ret::contains, () -> randomFrom(versions)));
300+
}
301+
return ret.stream();
302+
}
227303
}

0 commit comments

Comments
 (0)