Skip to content

Commit a547379

Browse files
kkrik-eskkrik-es
committed
Enable large numeric blocks for TSDB
1 parent 30f327e commit a547379

File tree

6 files changed

+71
-13
lines changed

6 files changed

+71
-13
lines changed

‎server/src/main/java/org/elasticsearch/common/settings/IndexScopedSettings.java‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -208,6 +208,7 @@ public final class IndexScopedSettings extends AbstractScopedSettings {
208208
IndexSettings.INDEX_MAPPER_SOURCE_MODE_SETTING,
209209
IndexSettings.RECOVERY_USE_SYNTHETIC_SOURCE_SETTING,
210210
IndexSettings.USE_TIME_SERIES_DOC_VALUES_FORMAT_SETTING,
211+
IndexSettings.USE_LOGS_DOC_VALUES_FORMAT_SETTING,
211212
InferenceMetadataFieldsMapper.USE_LEGACY_SEMANTIC_TEXT_FORMAT,
212213
IndexSettings.USE_ES_812_POSTINGS_FORMAT,
213214

‎server/src/main/java/org/elasticsearch/index/IndexMode.java‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -243,7 +243,7 @@ public SourceFieldMapper.Mode defaultSourceMode() {
243243
}
244244

245245
@Override
246-
public boolean useTimeSeriesDocValuesCodec() {
246+
public boolean useCustomDocValuesCodec() {
247247
return true;
248248
}
249249

@@ -338,7 +338,7 @@ public String getDefaultCodec() {
338338
}
339339

340340
@Override
341-
public boolean useTimeSeriesDocValuesCodec() {
341+
public boolean useCustomDocValuesCodec() {
342342
return true;
343343
}
344344

@@ -588,9 +588,9 @@ public boolean useEs812PostingsFormat() {
588588
}
589589

590590
/**
591-
* Whether by default to use the time series doc values codec.
591+
* Whether by default to use the custom doc values codec.
592592
*/
593-
public boolean useTimeSeriesDocValuesCodec() {
593+
public boolean useCustomDocValuesCodec() {
594594
return false;
595595
}
596596

‎server/src/main/java/org/elasticsearch/index/IndexSettings.java‎

Lines changed: 23 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -835,7 +835,20 @@ public Iterator<Setting<?>> settings() {
835835
return Boolean.FALSE.toString();
836836
}
837837
var indexMode = IndexSettings.MODE.get(settings);
838-
return Boolean.toString(indexMode.useTimeSeriesDocValuesCodec());
838+
return Boolean.toString(indexMode == IndexMode.TIME_SERIES && indexMode.useCustomDocValuesCodec());
839+
},
840+
Property.IndexScope,
841+
Property.Final
842+
);
843+
844+
public static final Setting<Boolean> USE_LOGS_DOC_VALUES_FORMAT_SETTING = Setting.boolSetting(
845+
"index.use_logs_doc_values_format",
846+
settings -> {
847+
if (settings == null) {
848+
return Boolean.FALSE.toString();
849+
}
850+
var indexMode = IndexSettings.MODE.get(settings);
851+
return Boolean.toString(indexMode == IndexMode.LOGSDB && indexMode.useCustomDocValuesCodec());
839852
},
840853
Property.IndexScope,
841854
Property.Final
@@ -1019,6 +1032,7 @@ private void setRetentionLeaseMillis(final TimeValue retentionLease) {
10191032
private final boolean useDocValuesSkipper;
10201033
private final boolean useTimeSeriesSyntheticId;
10211034
private final boolean useTimeSeriesDocValuesFormat;
1035+
private final boolean useLogsDocValuesFormat;
10221036
private final boolean useEs812PostingsFormat;
10231037

10241038
/**
@@ -1207,6 +1221,7 @@ public IndexSettings(final IndexMetadata indexMetadata, final Settings nodeSetti
12071221
useDocValuesSkipper = DOC_VALUES_SKIPPER && scopedSettings.get(USE_DOC_VALUES_SKIPPER);
12081222
seqNoIndexOptions = scopedSettings.get(SEQ_NO_INDEX_OPTIONS_SETTING);
12091223
useTimeSeriesDocValuesFormat = scopedSettings.get(USE_TIME_SERIES_DOC_VALUES_FORMAT_SETTING);
1224+
useLogsDocValuesFormat = scopedSettings.get(USE_LOGS_DOC_VALUES_FORMAT_SETTING);
12101225
useEs812PostingsFormat = scopedSettings.get(USE_ES_812_POSTINGS_FORMAT);
12111226
final var useSyntheticId = IndexSettings.TSDB_SYNTHETIC_ID_FEATURE_FLAG && scopedSettings.get(USE_SYNTHETIC_ID);
12121227
if (indexMetadata.useTimeSeriesSyntheticId() != useSyntheticId) {
@@ -1976,6 +1991,13 @@ public boolean useTimeSeriesDocValuesFormat() {
19761991
return useTimeSeriesDocValuesFormat;
19771992
}
19781993

1994+
/**
1995+
* @return Whether the logs doc value format should be used.
1996+
*/
1997+
public boolean useLogsDocValuesFormat() {
1998+
return useLogsDocValuesFormat;
1999+
}
2000+
19792001
/**
19802002
* @return Whether the ES 8.12 postings format should be used.
19812003
*/

‎server/src/main/java/org/elasticsearch/index/codec/PerFieldFormatSupplier.java‎

Lines changed: 18 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@
1515
import org.apache.lucene.codecs.lucene90.Lucene90DocValuesFormat;
1616
import org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsFormat;
1717
import org.elasticsearch.common.util.BigArrays;
18+
import org.elasticsearch.common.util.FeatureFlag;
1819
import org.elasticsearch.index.IndexMode;
1920
import org.elasticsearch.index.IndexSettings;
2021
import org.elasticsearch.index.IndexVersions;
@@ -43,6 +44,8 @@ public class PerFieldFormatSupplier {
4344
private static final Set<String> INCLUDE_META_FIELDS;
4445
private static final Set<String> EXCLUDE_MAPPER_TYPES;
4546

47+
private static final boolean TSDB_USE_LARGE_NUMERIC_BLOCKS = new FeatureFlag("tsdb_large_numeric_blocks").isEnabled();
48+
4649
static {
4750
// TODO: should we just allow all fields to use tsdb doc values codec?
4851
// Avoid using tsdb codec for fields like _seq_no, _primary_term.
@@ -59,7 +62,8 @@ public class PerFieldFormatSupplier {
5962

6063
private static final DocValuesFormat docValuesFormat = new Lucene90DocValuesFormat();
6164
private static final KnnVectorsFormat knnVectorsFormat = new Lucene99HnswVectorsFormat();
62-
private static final ES819TSDBDocValuesFormat tsdbDocValuesFormat = new ES819TSDBDocValuesFormat();
65+
private static final ES819TSDBDocValuesFormat tsdbDocValuesFormat = ES819TSDBDocValuesFormat.getInstance(TSDB_USE_LARGE_NUMERIC_BLOCKS);
66+
private static final ES819TSDBDocValuesFormat logsDocValuesFormat = ES819TSDBDocValuesFormat.getInstance(false);
6367
private static final ES812PostingsFormat es812PostingsFormat = new ES812PostingsFormat();
6468
private static final PostingsFormat completionPostingsFormat = PostingsFormat.forName("Completion101");
6569

@@ -139,6 +143,9 @@ public DocValuesFormat getDocValuesFormatForField(String field) {
139143
if (useTSDBDocValuesFormat(field)) {
140144
return tsdbDocValuesFormat;
141145
}
146+
if (useLogsDocValuesFormat(field)) {
147+
return logsDocValuesFormat;
148+
}
142149
return docValuesFormat;
143150
}
144151

@@ -156,6 +163,16 @@ boolean useTSDBDocValuesFormat(final String field) {
156163
&& mapperService.getIndexSettings().isES87TSDBCodecEnabled();
157164
}
158165

166+
boolean useLogsDocValuesFormat(final String field) {
167+
if (excludeFields(field)) {
168+
return false;
169+
}
170+
171+
return mapperService != null
172+
&& mapperService.getIndexSettings().useLogsDocValuesFormat()
173+
&& mapperService.getIndexSettings().isES87TSDBCodecEnabled();
174+
}
175+
159176
private boolean excludeFields(String fieldName) {
160177
return fieldName.startsWith("_") && INCLUDE_META_FIELDS.contains(fieldName) == false;
161178
}

‎server/src/main/java/org/elasticsearch/index/codec/tsdb/es819/ES819TSDBDocValuesFormat.java‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -137,6 +137,10 @@ private static boolean getOptimizedMergeEnabledDefault() {
137137
final BinaryDVCompressionMode binaryDVCompressionMode;
138138
final boolean enablePerBlockCompression;
139139

140+
public static ES819TSDBDocValuesFormat getInstance(boolean useLargeNumericBlock) {
141+
return useLargeNumericBlock ? new ES819TSDBDocValuesFormat(NUMERIC_LARGE_BLOCK_SHIFT) : new ES819TSDBDocValuesFormat();
142+
}
143+
140144
public ES819TSDBDocValuesFormat() {
141145
this(NUMERIC_BLOCK_SHIFT);
142146
}
@@ -200,7 +204,6 @@ public ES819TSDBDocValuesFormat(
200204
final int numericBlockShift
201205
) {
202206
super(CODEC_NAME);
203-
assert numericBlockShift == NUMERIC_BLOCK_SHIFT || numericBlockShift == NUMERIC_LARGE_BLOCK_SHIFT : numericBlockShift;
204207
if (skipIndexIntervalSize < 2) {
205208
throw new IllegalArgumentException("skipIndexIntervalSize must be > 1, got [" + skipIndexIntervalSize + "]");
206209
}

‎server/src/test/java/org/elasticsearch/index/codec/PerFieldMapperCodecTests.java‎

Lines changed: 21 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -156,6 +156,7 @@ public void testUseEs812PostingsFormat() throws IOException {
156156
public void testUseES87TSDBEncodingForTimestampField() throws IOException {
157157
PerFieldFormatSupplier perFieldMapperCodec = createFormatSupplier(true, true, true);
158158
assertThat((perFieldMapperCodec.useTSDBDocValuesFormat("@timestamp")), is(true));
159+
assertThat((perFieldMapperCodec.useLogsDocValuesFormat("@timestamp")), is(false));
159160
}
160161

161162
public void testDoNotUseES87TSDBEncodingForTimestampFieldNonTimeSeriesIndex() throws IOException {
@@ -167,6 +168,8 @@ public void testEnableES87TSDBCodec() throws IOException {
167168
PerFieldFormatSupplier perFieldMapperCodec = createFormatSupplier(true, false, IndexMode.TIME_SERIES, MAPPING_1);
168169
assertThat((perFieldMapperCodec.useTSDBDocValuesFormat("gauge")), is(true));
169170
assertThat((perFieldMapperCodec.useTSDBDocValuesFormat("@timestamp")), is(true));
171+
assertThat((perFieldMapperCodec.useLogsDocValuesFormat("gauge")), is(false));
172+
assertThat((perFieldMapperCodec.useLogsDocValuesFormat("@timestamp")), is(false));
170173
}
171174

172175
public void testDisableES87TSDBCodec() throws IOException {
@@ -223,34 +226,46 @@ public void testUseTimeSeriesDocValuesCodecSetting() throws IOException {
223226
assertThat((perFieldMapperCodec.useTSDBDocValuesFormat("@timestamp")), is(true));
224227
assertThat((perFieldMapperCodec.useTSDBDocValuesFormat("counter")), is(true));
225228
assertThat((perFieldMapperCodec.useTSDBDocValuesFormat("gauge")), is(true));
229+
assertThat((perFieldMapperCodec.useLogsDocValuesFormat("@timestamp")), is(false));
230+
assertThat((perFieldMapperCodec.useLogsDocValuesFormat("counter")), is(false));
231+
assertThat((perFieldMapperCodec.useLogsDocValuesFormat("gauge")), is(false));
226232
}
227233

228234
public void testUseTimeSeriesModeAndCodecEnabled() throws IOException {
229235
PerFieldFormatSupplier perFieldMapperCodec = createFormatSupplier(true, false, IndexMode.TIME_SERIES, MAPPING_2);
230236
assertThat((perFieldMapperCodec.useTSDBDocValuesFormat("@timestamp")), is(true));
231237
assertThat((perFieldMapperCodec.useTSDBDocValuesFormat("counter")), is(true));
232238
assertThat((perFieldMapperCodec.useTSDBDocValuesFormat("gauge")), is(true));
239+
assertThat((perFieldMapperCodec.useLogsDocValuesFormat("@timestamp")), is(false));
240+
assertThat((perFieldMapperCodec.useLogsDocValuesFormat("counter")), is(false));
241+
assertThat((perFieldMapperCodec.useLogsDocValuesFormat("gauge")), is(false));
233242
}
234243

235244
public void testLogsIndexMode() throws IOException {
236245
PerFieldFormatSupplier perFieldMapperCodec = createFormatSupplier(IndexMode.LOGSDB, MAPPING_3);
237-
assertThat((perFieldMapperCodec.useTSDBDocValuesFormat("@timestamp")), is(true));
238-
assertThat((perFieldMapperCodec.useTSDBDocValuesFormat("hostname")), is(true));
239-
assertThat((perFieldMapperCodec.useTSDBDocValuesFormat("response_size")), is(true));
246+
assertThat((perFieldMapperCodec.useTSDBDocValuesFormat("@timestamp")), is(false));
247+
assertThat((perFieldMapperCodec.useTSDBDocValuesFormat("hostname")), is(false));
248+
assertThat((perFieldMapperCodec.useTSDBDocValuesFormat("response_size")), is(false));
249+
assertThat((perFieldMapperCodec.useLogsDocValuesFormat("@timestamp")), is(true));
250+
assertThat((perFieldMapperCodec.useLogsDocValuesFormat("hostname")), is(true));
251+
assertThat((perFieldMapperCodec.useLogsDocValuesFormat("response_size")), is(true));
240252
}
241253

242254
public void testMetaFields() throws IOException {
243255
PerFieldFormatSupplier perFieldMapperCodec = createFormatSupplier(IndexMode.LOGSDB, MAPPING_3);
244-
assertThat((perFieldMapperCodec.useTSDBDocValuesFormat(TimeSeriesIdFieldMapper.NAME)), is(true));
245-
assertThat((perFieldMapperCodec.useTSDBDocValuesFormat(TimeSeriesRoutingHashFieldMapper.NAME)), is(true));
256+
assertThat((perFieldMapperCodec.useTSDBDocValuesFormat(TimeSeriesIdFieldMapper.NAME)), is(false));
257+
assertThat((perFieldMapperCodec.useTSDBDocValuesFormat(TimeSeriesRoutingHashFieldMapper.NAME)), is(false));
258+
assertThat((perFieldMapperCodec.useLogsDocValuesFormat(TimeSeriesIdFieldMapper.NAME)), is(true));
259+
assertThat((perFieldMapperCodec.useLogsDocValuesFormat(TimeSeriesRoutingHashFieldMapper.NAME)), is(true));
246260
// See: PerFieldFormatSupplier why these fields shouldn't use tsdb codec
247261
assertThat((perFieldMapperCodec.useTSDBDocValuesFormat(SourceFieldMapper.RECOVERY_SOURCE_NAME)), is(false));
248262
assertThat((perFieldMapperCodec.useTSDBDocValuesFormat(SourceFieldMapper.RECOVERY_SOURCE_SIZE_NAME)), is(false));
249263
}
250264

251265
public void testSeqnoField() throws IOException {
252266
PerFieldFormatSupplier perFieldMapperCodec = createFormatSupplier(IndexMode.LOGSDB, MAPPING_3);
253-
assertThat((perFieldMapperCodec.useTSDBDocValuesFormat(SeqNoFieldMapper.NAME)), is(true));
267+
assertThat((perFieldMapperCodec.useTSDBDocValuesFormat(SeqNoFieldMapper.NAME)), is(false));
268+
assertThat((perFieldMapperCodec.useLogsDocValuesFormat(SeqNoFieldMapper.NAME)), is(true));
254269
}
255270

256271
private PerFieldFormatSupplier createFormatSupplier(IndexMode mode, String mapping) throws IOException {

0 commit comments

Comments
 (0)