Skip to content

Commit 27d8274

Browse files
Kubik42Kubik42
authored
Added a ES812 postings format index setting (#137857)
* Added an index settings for using ES 812 postings format * Changed setting name
1 parent db5026b commit 27d8274

File tree

5 files changed

+87
-20
lines changed

5 files changed

+87
-20
lines changed

‎server/src/main/java/org/elasticsearch/common/settings/IndexScopedSettings.java‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -209,6 +209,7 @@ public final class IndexScopedSettings extends AbstractScopedSettings {
209209
IndexSettings.RECOVERY_USE_SYNTHETIC_SOURCE_SETTING,
210210
IndexSettings.USE_TIME_SERIES_DOC_VALUES_FORMAT_SETTING,
211211
InferenceMetadataFieldsMapper.USE_LEGACY_SEMANTIC_TEXT_FORMAT,
212+
IndexSettings.USE_ES_812_POSTINGS_FORMAT,
212213

213214
// validate that built-in similarities don't get redefined
214215
Setting.groupSetting("index.similarity.", (s) -> {

‎server/src/main/java/org/elasticsearch/index/IndexMode.java‎

Lines changed: 13 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -128,11 +128,6 @@ public void validateSourceFieldMapper(SourceFieldMapper sourceFieldMapper) {}
128128
public SourceFieldMapper.Mode defaultSourceMode() {
129129
return SourceFieldMapper.Mode.STORED;
130130
}
131-
132-
@Override
133-
public boolean useDefaultPostingsFormat() {
134-
return true;
135-
}
136131
},
137132
TIME_SERIES("time_series") {
138133
@Override
@@ -251,6 +246,11 @@ public SourceFieldMapper.Mode defaultSourceMode() {
251246
public boolean useTimeSeriesDocValuesCodec() {
252247
return true;
253248
}
249+
250+
@Override
251+
public boolean useEs812PostingsFormat() {
252+
return true;
253+
}
254254
},
255255
LOGSDB("logsdb") {
256256
@Override
@@ -341,6 +341,11 @@ public String getDefaultCodec() {
341341
public boolean useTimeSeriesDocValuesCodec() {
342342
return true;
343343
}
344+
345+
@Override
346+
public boolean useEs812PostingsFormat() {
347+
return true;
348+
}
344349
},
345350
LOOKUP("lookup") {
346351
@Override
@@ -575,9 +580,10 @@ public String getDefaultCodec() {
575580
}
576581

577582
/**
578-
* Whether the default posting format (for inverted indices) from Lucene should be used.
583+
* Whether by default to use the ES 8.12 {@link org.apache.lucene.codecs.PostingsFormat}. This is a historical PostingsFormat we used
584+
* for all indices by default. However, starting with Lucene 10.3, we began using a new, more modern format, for standard indices.
579585
*/
580-
public boolean useDefaultPostingsFormat() {
586+
public boolean useEs812PostingsFormat() {
581587
return false;
582588
}
583589

‎server/src/main/java/org/elasticsearch/index/IndexSettings.java‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -855,6 +855,14 @@ public Iterator<Setting<?>> settings() {
855855
Property.IndexSettingDeprecatedInV7AndRemovedInV8
856856
);
857857

858+
public static final Setting<Boolean> USE_ES_812_POSTINGS_FORMAT = Setting.boolSetting("index.use_legacy_postings_format", settings -> {
859+
if (settings == null) {
860+
return Boolean.FALSE.toString();
861+
}
862+
IndexMode indexMode = IndexSettings.MODE.get(settings);
863+
return Boolean.toString(indexMode.useEs812PostingsFormat());
864+
}, Property.IndexScope, Property.Final);
865+
858866
/**
859867
* The `index.mapping.ignore_above` setting defines the maximum length for the content of a field that will be indexed
860868
* or stored. If the length of the field’s content exceeds this limit, the field value will be ignored during indexing.
@@ -1011,6 +1019,7 @@ private void setRetentionLeaseMillis(final TimeValue retentionLease) {
10111019
private final boolean useDocValuesSkipper;
10121020
private final boolean useTimeSeriesSyntheticId;
10131021
private final boolean useTimeSeriesDocValuesFormat;
1022+
private final boolean useEs812PostingsFormat;
10141023

10151024
/**
10161025
* The maximum number of refresh listeners allows on this shard.
@@ -1198,6 +1207,7 @@ public IndexSettings(final IndexMetadata indexMetadata, final Settings nodeSetti
11981207
useDocValuesSkipper = DOC_VALUES_SKIPPER && scopedSettings.get(USE_DOC_VALUES_SKIPPER);
11991208
seqNoIndexOptions = scopedSettings.get(SEQ_NO_INDEX_OPTIONS_SETTING);
12001209
useTimeSeriesDocValuesFormat = scopedSettings.get(USE_TIME_SERIES_DOC_VALUES_FORMAT_SETTING);
1210+
useEs812PostingsFormat = scopedSettings.get(USE_ES_812_POSTINGS_FORMAT);
12011211
final var useSyntheticId = IndexSettings.TSDB_SYNTHETIC_ID_FEATURE_FLAG && scopedSettings.get(USE_SYNTHETIC_ID);
12021212
if (indexMetadata.useTimeSeriesSyntheticId() != useSyntheticId) {
12031213
assert false;
@@ -1966,6 +1976,13 @@ public boolean useTimeSeriesDocValuesFormat() {
19661976
return useTimeSeriesDocValuesFormat;
19671977
}
19681978

1979+
/**
1980+
* @return Whether the ES 8.12 postings format should be used.
1981+
*/
1982+
public boolean useEs812PostingsFormat() {
1983+
return useEs812PostingsFormat;
1984+
}
1985+
19691986
/**
19701987
* The bounds for {@code @timestamp} on this index or
19711988
* {@code null} if there are no bounds.

‎server/src/main/java/org/elasticsearch/index/codec/PerFieldFormatSupplier.java‎

Lines changed: 12 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -69,14 +69,21 @@ public class PerFieldFormatSupplier {
6969
public PerFieldFormatSupplier(MapperService mapperService, BigArrays bigArrays) {
7070
this.mapperService = mapperService;
7171
this.bloomFilterPostingsFormat = new ES87BloomFilterPostingsFormat(bigArrays, this::internalGetPostingsFormatForField);
72+
this.defaultPostingsFormat = getDefaultPostingsFormat(mapperService);
73+
}
7274

75+
private static PostingsFormat getDefaultPostingsFormat(final MapperService mapperService) {
76+
// we migrated to using a new postings format for the standard indices with Lucene 10.3
7377
if (mapperService != null
74-
&& mapperService.getIndexSettings().getIndexVersionCreated().onOrAfter(IndexVersions.UPGRADE_TO_LUCENE_10_3_0)
75-
&& mapperService.getIndexSettings().getMode().useDefaultPostingsFormat()) {
76-
defaultPostingsFormat = Elasticsearch92Lucene103Codec.DEFAULT_POSTINGS_FORMAT;
78+
&& mapperService.getIndexSettings().getIndexVersionCreated().onOrAfter(IndexVersions.UPGRADE_TO_LUCENE_10_3_0)) {
79+
if (IndexSettings.USE_ES_812_POSTINGS_FORMAT.get(mapperService.getIndexSettings().getSettings())) {
80+
return es812PostingsFormat;
81+
} else {
82+
return Elasticsearch92Lucene103Codec.DEFAULT_POSTINGS_FORMAT;
83+
}
7784
} else {
78-
// our own posting format using PFOR
79-
defaultPostingsFormat = es812PostingsFormat;
85+
// our own posting format using PFOR, used for logsdb and tsdb indices by default
86+
return es812PostingsFormat;
8087
}
8188
}
8289

‎server/src/test/java/org/elasticsearch/index/codec/PerFieldMapperCodecTests.java‎

Lines changed: 44 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -126,6 +126,33 @@ public void testUseBloomFilterWithTimestampFieldEnabled_disableBloomFilter() thr
126126
);
127127
}
128128

129+
public void testUseEs812PostingsFormat() throws IOException {
130+
PerFieldFormatSupplier perFieldMapperCodec;
131+
132+
// standard index mode
133+
perFieldMapperCodec = createFormatSupplier(false, false, IndexMode.STANDARD, MAPPING_1);
134+
assertThat(perFieldMapperCodec.getPostingsFormatForField("gauge"), instanceOf(Lucene103PostingsFormat.class));
135+
136+
perFieldMapperCodec = createFormatSupplier(false, true, IndexMode.STANDARD, MAPPING_1);
137+
assertThat(perFieldMapperCodec.getPostingsFormatForField("gauge"), instanceOf(ES812PostingsFormat.class));
138+
139+
// LogsDB index mode
140+
// by default, logsdb uses the ES 8.12 postings format
141+
perFieldMapperCodec = createFormatSupplier(false, false, IndexMode.LOGSDB, MAPPING_3);
142+
assertThat(perFieldMapperCodec.getPostingsFormatForField("message"), instanceOf(ES812PostingsFormat.class));
143+
144+
perFieldMapperCodec = createFormatSupplier(false, true, IndexMode.LOGSDB, MAPPING_3);
145+
assertThat(perFieldMapperCodec.getPostingsFormatForField("message"), instanceOf(ES812PostingsFormat.class));
146+
147+
// time series index mode
148+
// by default, logsdb uses the ES 8.12 postings format
149+
perFieldMapperCodec = createFormatSupplier(false, false, IndexMode.TIME_SERIES, MAPPING_1);
150+
assertThat(perFieldMapperCodec.getPostingsFormatForField("gauge"), instanceOf(ES812PostingsFormat.class));
151+
152+
perFieldMapperCodec = createFormatSupplier(false, true, IndexMode.TIME_SERIES, MAPPING_1);
153+
assertThat(perFieldMapperCodec.getPostingsFormatForField("gauge"), instanceOf(ES812PostingsFormat.class));
154+
}
155+
129156
public void testUseES87TSDBEncodingForTimestampField() throws IOException {
130157
PerFieldFormatSupplier perFieldMapperCodec = createFormatSupplier(true, true, true);
131158
assertThat((perFieldMapperCodec.useTSDBDocValuesFormat("@timestamp")), is(true));
@@ -137,13 +164,13 @@ public void testDoNotUseES87TSDBEncodingForTimestampFieldNonTimeSeriesIndex() th
137164
}
138165

139166
public void testEnableES87TSDBCodec() throws IOException {
140-
PerFieldFormatSupplier perFieldMapperCodec = createFormatSupplier(true, IndexMode.TIME_SERIES, MAPPING_1);
167+
PerFieldFormatSupplier perFieldMapperCodec = createFormatSupplier(true, false, IndexMode.TIME_SERIES, MAPPING_1);
141168
assertThat((perFieldMapperCodec.useTSDBDocValuesFormat("gauge")), is(true));
142169
assertThat((perFieldMapperCodec.useTSDBDocValuesFormat("@timestamp")), is(true));
143170
}
144171

145172
public void testDisableES87TSDBCodec() throws IOException {
146-
PerFieldFormatSupplier perFieldMapperCodec = createFormatSupplier(false, IndexMode.TIME_SERIES, MAPPING_1);
173+
PerFieldFormatSupplier perFieldMapperCodec = createFormatSupplier(false, false, IndexMode.TIME_SERIES, MAPPING_1);
147174
assertThat((perFieldMapperCodec.useTSDBDocValuesFormat("gauge")), is(false));
148175
assertThat((perFieldMapperCodec.useTSDBDocValuesFormat("@timestamp")), is(false));
149176
}
@@ -178,7 +205,7 @@ private PerFieldFormatSupplier createFormatSupplier(boolean timestampField, bool
178205
}
179206

180207
public void testUseES87TSDBEncodingSettingDisabled() throws IOException {
181-
PerFieldFormatSupplier perFieldMapperCodec = createFormatSupplier(false, IndexMode.TIME_SERIES, MAPPING_2);
208+
PerFieldFormatSupplier perFieldMapperCodec = createFormatSupplier(false, false, IndexMode.TIME_SERIES, MAPPING_2);
182209
assertThat((perFieldMapperCodec.useTSDBDocValuesFormat("@timestamp")), is(false));
183210
assertThat((perFieldMapperCodec.useTSDBDocValuesFormat("counter")), is(false));
184211
assertThat((perFieldMapperCodec.useTSDBDocValuesFormat("gauge")), is(false));
@@ -192,14 +219,14 @@ public void testUseTimeSeriesModeDisabledCodecDisabled() throws IOException {
192219
}
193220

194221
public void testUseTimeSeriesDocValuesCodecSetting() throws IOException {
195-
PerFieldFormatSupplier perFieldMapperCodec = createFormatSupplier(true, null, IndexMode.STANDARD, MAPPING_2);
222+
PerFieldFormatSupplier perFieldMapperCodec = createFormatSupplier(true, null, false, IndexMode.STANDARD, MAPPING_2);
196223
assertThat((perFieldMapperCodec.useTSDBDocValuesFormat("@timestamp")), is(true));
197224
assertThat((perFieldMapperCodec.useTSDBDocValuesFormat("counter")), is(true));
198225
assertThat((perFieldMapperCodec.useTSDBDocValuesFormat("gauge")), is(true));
199226
}
200227

201228
public void testUseTimeSeriesModeAndCodecEnabled() throws IOException {
202-
PerFieldFormatSupplier perFieldMapperCodec = createFormatSupplier(true, IndexMode.TIME_SERIES, MAPPING_2);
229+
PerFieldFormatSupplier perFieldMapperCodec = createFormatSupplier(true, false, IndexMode.TIME_SERIES, MAPPING_2);
203230
assertThat((perFieldMapperCodec.useTSDBDocValuesFormat("@timestamp")), is(true));
204231
assertThat((perFieldMapperCodec.useTSDBDocValuesFormat("counter")), is(true));
205232
assertThat((perFieldMapperCodec.useTSDBDocValuesFormat("gauge")), is(true));
@@ -227,16 +254,22 @@ public void testSeqnoField() throws IOException {
227254
}
228255

229256
private PerFieldFormatSupplier createFormatSupplier(IndexMode mode, String mapping) throws IOException {
230-
return createFormatSupplier(null, mode, mapping);
257+
return createFormatSupplier(null, false, mode, mapping);
231258
}
232259

233-
private PerFieldFormatSupplier createFormatSupplier(Boolean enableES87TSDBCodec, IndexMode mode, String mapping) throws IOException {
234-
return createFormatSupplier(null, enableES87TSDBCodec, mode, mapping);
260+
private PerFieldFormatSupplier createFormatSupplier(
261+
Boolean enableES87TSDBCodec,
262+
Boolean useEs812PostingsFormat,
263+
IndexMode mode,
264+
String mapping
265+
) throws IOException {
266+
return createFormatSupplier(null, enableES87TSDBCodec, useEs812PostingsFormat, mode, mapping);
235267
}
236268

237269
private PerFieldFormatSupplier createFormatSupplier(
238270
Boolean useTimeSeriesDocValuesFormatSetting,
239271
Boolean enableES87TSDBCodec,
272+
Boolean useEs812PostingsFormat,
240273
IndexMode mode,
241274
String mapping
242275
) throws IOException {
@@ -251,6 +284,9 @@ private PerFieldFormatSupplier createFormatSupplier(
251284
if (useTimeSeriesDocValuesFormatSetting != null) {
252285
settings.put(IndexSettings.USE_TIME_SERIES_DOC_VALUES_FORMAT_SETTING.getKey(), useTimeSeriesDocValuesFormatSetting);
253286
}
287+
if (useEs812PostingsFormat) {
288+
settings.put(IndexSettings.USE_ES_812_POSTINGS_FORMAT.getKey(), true);
289+
}
254290
MapperService mapperService = MapperTestUtils.newMapperService(xContentRegistry(), createTempDir(), settings.build(), "test");
255291
mapperService.merge("type", new CompressedXContent(mapping), MapperService.MergeReason.MAPPING_UPDATE);
256292
return new PerFieldFormatSupplier(mapperService, BigArrays.NON_RECYCLING_INSTANCE);

0 commit comments

Comments
 (0)