Skip to content

Commit cab22b3

Browse files
ChrisHegartyChrisHegarty
authored
[8.19.1] Fix default missing index sort value of data_nanos pre 7.14 (#132172)
This commit fixes an issue whereby indices created prior to 7.14 that have an index sort on a date_nanos field can no longer be opened (with versions >= 7.14). When opening an index the configured index sort, derived from the index settings, must match exactly that of the sort encoded in the index itself. A change to fix a bug back in 7.14 changed this for date_nanos fields whose index.sort.missing value is absent, see #74760. Specifically, the default minimum value changed from Long.MIN_VALUE to 0L. The change in this commit restores the default minimum value for indices prior to 7.14.
1 parent 25dea9b commit cab22b3

File tree

4 files changed

+122
-3
lines changed

4 files changed

+122
-3
lines changed

‎docs/changelog/132162.yaml

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
pr: 132162
2+
summary: Fix default missing index sort value of `data_nanos` pre 7.14
3+
area: Search
4+
type: bug
5+
issues:
6+
- 132040

‎server/src/internalClusterTest/java/org/elasticsearch/index/IndexSortIT.java

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@
1111

1212
import org.apache.lucene.search.Sort;
1313
import org.apache.lucene.search.SortField;
14+
import org.apache.lucene.search.SortedNumericSelector;
1415
import org.apache.lucene.search.SortedNumericSortField;
1516
import org.apache.lucene.search.SortedSetSortField;
1617
import org.elasticsearch.common.settings.Settings;
@@ -80,6 +81,33 @@ public void testIndexSort() {
8081
assertSortedSegments("test", indexSort);
8182
}
8283

84+
public void testIndexSortDateNanos() {
85+
prepareCreate("test").setSettings(
86+
Settings.builder()
87+
.put(indexSettings())
88+
.put("index.number_of_shards", "1")
89+
.put("index.number_of_replicas", "1")
90+
.put("index.sort.field", "@timestamp")
91+
.put("index.sort.order", "desc")
92+
).setMapping("""
93+
{
94+
"properties": {
95+
"@timestamp": {
96+
"type": "date_nanos"
97+
}
98+
}
99+
}
100+
""").get();
101+
102+
flushAndRefresh();
103+
ensureYellow();
104+
105+
SortField sf = new SortedNumericSortField("@timestamp", SortField.Type.LONG, true, SortedNumericSelector.Type.MAX);
106+
sf.setMissingValue(0L);
107+
Sort expectedIndexSort = new Sort(sf);
108+
assertSortedSegments("test", expectedIndexSort);
109+
}
110+
83111
public void testInvalidIndexSort() {
84112
IllegalArgumentException exc = expectThrows(
85113
IllegalArgumentException.class,

‎server/src/main/java/org/elasticsearch/index/fielddata/IndexNumericFieldData.java

Lines changed: 12 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -144,10 +144,19 @@ public SortField sortField(
144144
boolean reverse
145145
) {
146146
SortField sortField = sortField(missingValue, sortMode, nested, reverse);
147-
if (indexCreatedVersion.onOrAfter(IndexVersions.INDEX_INT_SORT_INT_TYPE_8_19)
148-
|| getNumericType().sortFieldType != SortField.Type.INT) {
147+
if (getNumericType() == NumericType.DATE_NANOSECONDS
148+
&& indexCreatedVersion.before(IndexVersions.V_7_14_0)
149+
&& missingValue == null
150+
&& Long.valueOf(0L).equals(sortField.getMissingValue())) {
151+
// 7.14 changed the default missing value of sort on date_nanos, from Long.MIN_VALUE
152+
// to 0L - for compatibility we require to a missing value of MIN_VALUE to allow to
153+
// open the index.
154+
sortField.setMissingValue(Long.MIN_VALUE);
149155
return sortField;
150-
}
156+
} else if (indexCreatedVersion.onOrAfter(IndexVersions.INDEX_INT_SORT_INT_TYPE_8_19)
157+
|| getNumericType().sortFieldType != SortField.Type.INT) {
158+
return sortField;
159+
}
151160
if ((sortField instanceof SortedNumericSortField) == false) {
152161
return sortField;
153162
}

‎server/src/test/java/org/elasticsearch/index/IndexSortSettingsTests.java

Lines changed: 76 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@
1313
import org.apache.lucene.search.Sort;
1414
import org.elasticsearch.cluster.metadata.IndexMetadata;
1515
import org.elasticsearch.common.settings.Settings;
16+
import org.elasticsearch.common.time.DateUtils;
1617
import org.elasticsearch.common.util.Maps;
1718
import org.elasticsearch.index.fielddata.FieldDataContext;
1819
import org.elasticsearch.index.fielddata.IndexFieldData;
@@ -30,9 +31,12 @@
3031
import org.elasticsearch.search.sort.SortOrder;
3132
import org.elasticsearch.test.ESTestCase;
3233

34+
import java.util.ArrayList;
3335
import java.util.Collections;
36+
import java.util.List;
3437
import java.util.Map;
3538
import java.util.Set;
39+
import java.util.stream.Stream;
3640

3741
import static org.elasticsearch.index.IndexSettingsTests.newIndexMeta;
3842
import static org.hamcrest.Matchers.arrayWithSize;
@@ -174,6 +178,58 @@ public void testSortingAgainstAliasesPre713() {
174178
);
175179
}
176180

181+
public void testSortMissingValueDateNanoFieldPre714() {
182+
MappedFieldType tsField = new DateFieldMapper.DateFieldType("@timestamp", true, DateFieldMapper.Resolution.NANOSECONDS);
183+
var indexSettingsBuilder = Settings.builder();
184+
indexSettingsBuilder.put("index.sort.field", "@timestamp");
185+
indexSettingsBuilder.put("index.sort.order", "desc");
186+
187+
// test with index version 7.13 and before
188+
var pre714Versions = Stream.concat(Stream.of(IndexVersions.V_7_13_0), randomVersionsBefore(IndexVersions.V_7_13_0)).toList();
189+
for (var version : pre714Versions) {
190+
indexSettingsBuilder.put(IndexMetadata.SETTING_VERSION_CREATED, version);
191+
Sort sort = buildIndexSort(indexSettings(indexSettingsBuilder.build()), Map.of("@timestamp", tsField));
192+
assertThat(sort.getSort(), arrayWithSize(1));
193+
assertThat(sort.getSort()[0].getField(), equalTo("@timestamp"));
194+
assertThat(sort.getSort()[0].getMissingValue(), equalTo(Long.MIN_VALUE));
195+
}
196+
197+
// now test with index version 7.14 and after
198+
var post713Versions = Stream.concat(Stream.of(IndexVersions.V_7_14_0), randomVersionsAfter(IndexVersions.V_7_14_0)).toList();
199+
for (var version : post713Versions) {
200+
indexSettingsBuilder.put(IndexMetadata.SETTING_VERSION_CREATED, version);
201+
Sort sort = buildIndexSort(indexSettings(indexSettingsBuilder.build()), Map.of("@timestamp", tsField));
202+
assertThat(sort.getSort(), arrayWithSize(1));
203+
assertThat(sort.getSort()[0].getField(), equalTo("@timestamp"));
204+
assertThat(sort.getSort()[0].getMissingValue(), equalTo(0L));
205+
}
206+
207+
// asc order has not changed behaviour in any version
208+
indexSettingsBuilder.put("index.sort.order", "asc");
209+
var allVersions = Stream.concat(post713Versions.stream(), pre714Versions.stream()).toList();
210+
for (var version : allVersions) {
211+
indexSettingsBuilder.put(IndexMetadata.SETTING_VERSION_CREATED, version);
212+
Sort sort = buildIndexSort(indexSettings(indexSettingsBuilder.build()), Map.of("@timestamp", tsField));
213+
assertThat(sort.getSort(), arrayWithSize(1));
214+
assertThat(sort.getSort()[0].getField(), equalTo("@timestamp"));
215+
assertThat(sort.getSort()[0].getMissingValue(), equalTo(DateUtils.MAX_NANOSECOND));
216+
}
217+
218+
// ensure no change in behaviour when a missing value is set
219+
indexSettingsBuilder.put("index.sort.missing", "_first");
220+
for (var version : allVersions) {
221+
indexSettingsBuilder.put(IndexMetadata.SETTING_VERSION_CREATED, version);
222+
Sort sort = buildIndexSort(indexSettings(indexSettingsBuilder.build()), Map.of("@timestamp", tsField));
223+
assertThat(sort.getSort()[0].getMissingValue(), equalTo(0L));
224+
}
225+
indexSettingsBuilder.put("index.sort.missing", "_last");
226+
for (var version : allVersions) {
227+
indexSettingsBuilder.put(IndexMetadata.SETTING_VERSION_CREATED, version);
228+
Sort sort = buildIndexSort(indexSettings(indexSettingsBuilder.build()), Map.of("@timestamp", tsField));
229+
assertThat(sort.getSort()[0].getMissingValue(), equalTo(Long.MAX_VALUE));
230+
}
231+
}
232+
177233
public void testTimeSeriesMode() {
178234
IndexSettings indexSettings = indexSettings(
179235
Settings.builder()
@@ -224,4 +280,24 @@ private Sort buildIndexSort(IndexSettings indexSettings, Map<String, MappedField
224280
)
225281
);
226282
}
283+
284+
/* Returns a stream of versions before the given version */
285+
Stream<IndexVersion> randomVersionsBefore(IndexVersion indexVersion) {
286+
var versions = IndexVersions.getAllVersions().stream().filter(v -> v.before(indexVersion)).toList();
287+
List<IndexVersion> ret = new ArrayList<>();
288+
for (int i = 0; i < 10; i++) {
289+
ret.add(randomValueOtherThanMany(ret::contains, () -> randomFrom(versions)));
290+
}
291+
return ret.stream();
292+
}
293+
294+
/* Returns a stream of versions after the given version */
295+
Stream<IndexVersion> randomVersionsAfter(IndexVersion indexVersion) {
296+
var versions = IndexVersions.getAllVersions().stream().filter(v -> v.after(indexVersion)).toList();
297+
List<IndexVersion> ret = new ArrayList<>();
298+
for (int i = 0; i < 10; i++) {
299+
ret.add(randomValueOtherThanMany(ret::contains, () -> randomFrom(versions)));
300+
}
301+
return ret.stream();
302+
}
227303
}

0 commit comments

Comments
 (0)