Skip to content

Commit 3de8878

Browse files
authored
Remove DOC_VALUES_SKIPPER feature flag (elastic#138723)
Removing this flag will enable skippers on keyword, ip, boolean, date and numeric fields that have index=false and doc_values=true when the `index.mapping.use_doc_values_skipper` setting is true. The setting defaults to `true` for time series indexes, and `false` for all other index modes.
1 parent bb68bd8 commit 3de8878

File tree

19 files changed

+119
-147
lines changed

19 files changed

+119
-147
lines changed

docs/changelog/138723.yaml

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
pr: 138723
2+
summary: Remove DOC_VALUES_SKIPPER feature flag
3+
area: Mapping
4+
type: feature
5+
issues: []
6+
highlight:
7+
title: Enable doc_values skippers
8+
body: |-
9+
Doc_values skippers add a sparse index to doc_values fields, allowing efficient
10+
querying and filtering on a field without having to build a separate BKD or terms
11+
index. These are now enabled automatically on any field configured with
12+
index=false and doc_values=true if the index setting `index.mapping.use_doc_values_skipper`
13+
is set to `true` (default `false`, or `true` for TSDB indexes).
14+
15+
TSDB indexes now default to using skippers in place of indexes for their
16+
@timestamp, dimension and _tsid fields, greatly reducing their on-disk
17+
footprint. To disable skippers in TSDB indexes, set `index.mapping.use_doc_values_skipper`
18+
to `false`.
19+
20+
notable: true

modules/data-streams/src/test/java/org/elasticsearch/datastreams/mapper/DataStreamTimestampFieldMapperTests.java

Lines changed: 22 additions & 114 deletions
Large diffs are not rendered by default.

qa/ccs-common-rest/src/yamlRestTest/java/org/elasticsearch/test/rest/yaml/CcsCommonYamlTestSuiteIT.java

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -101,8 +101,7 @@ public class CcsCommonYamlTestSuiteIT extends ESClientYamlSuiteTestCase {
101101
// geohex_grid requires gold license
102102
.setting("xpack.license.self_generated.type", "trial")
103103
.feature(FeatureFlag.TIME_SERIES_MODE)
104-
.feature(FeatureFlag.SYNTHETIC_VECTORS)
105-
.feature(FeatureFlag.DOC_VALUES_SKIPPER);
104+
.feature(FeatureFlag.SYNTHETIC_VECTORS);
106105

107106
private static ElasticsearchCluster remoteCluster = ElasticsearchCluster.local()
108107
.name(REMOTE_CLUSTER_NAME)

qa/ccs-common-rest/src/yamlRestTest/java/org/elasticsearch/test/rest/yaml/RcsCcsCommonYamlTestSuiteIT.java

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -101,7 +101,6 @@ public class RcsCcsCommonYamlTestSuiteIT extends ESClientYamlSuiteTestCase {
101101
.setting("xpack.security.remote_cluster_client.ssl.enabled", "false")
102102
.feature(FeatureFlag.TIME_SERIES_MODE)
103103
.feature(FeatureFlag.SYNTHETIC_VECTORS)
104-
.feature(FeatureFlag.DOC_VALUES_SKIPPER)
105104
.user("test_admin", "x-pack-test-password");
106105

107106
private static ElasticsearchCluster fulfillingCluster = ElasticsearchCluster.local()

qa/smoke-test-multinode/src/yamlRestTest/java/org/elasticsearch/smoketest/SmokeTestMultiNodeClientYamlTestSuiteIT.java

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -35,7 +35,6 @@ public class SmokeTestMultiNodeClientYamlTestSuiteIT extends ESClientYamlSuiteTe
3535
// The first node does not have the ingest role so we're sure ingest requests are forwarded:
3636
.node(0, n -> n.setting("node.roles", "[master,data,ml,remote_cluster_client,transform]"))
3737
.feature(FeatureFlag.TIME_SERIES_MODE)
38-
.feature(FeatureFlag.DOC_VALUES_SKIPPER)
3938
.feature(FeatureFlag.SYNTHETIC_VECTORS)
4039
.feature(FeatureFlag.RANDOM_SAMPLING)
4140
.build();

rest-api-spec/src/yamlRestTest/java/org/elasticsearch/test/rest/ClientYamlTestSuiteIT.java

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -35,7 +35,6 @@ public class ClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase {
3535
.module("health-shards-availability")
3636
.module("data-streams")
3737
.feature(FeatureFlag.TIME_SERIES_MODE)
38-
.feature(FeatureFlag.DOC_VALUES_SKIPPER)
3938
.feature(FeatureFlag.SYNTHETIC_VECTORS)
4039
.feature(FeatureFlag.RANDOM_SAMPLING)
4140
.feature(FeatureFlag.EXTENDED_DOC_VALUES_PARAMS)

server/src/internalClusterTest/java/org/elasticsearch/index/shard/SearchIdleIT.java

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -235,11 +235,10 @@ public void testSearchIdleStats() throws InterruptedException {
235235
}
236236

237237
public void testSearchIdleBoolQueryMatchOneIndex() throws InterruptedException {
238-
checkSearchIdleBoolQueryMatchOneIndex(IndexSettings.DOC_VALUES_SKIPPER);
238+
checkSearchIdleBoolQueryMatchOneIndex(true);
239239
}
240240

241241
public void testSearchIdleBoolQueryMatchOneIndexWithDocValuesSkipper() throws InterruptedException {
242-
assumeTrue("doc values skipper feature should be enabled", IndexSettings.DOC_VALUES_SKIPPER);
243242
checkSearchIdleBoolQueryMatchOneIndex(false);
244243
}
245244

server/src/main/java/org/elasticsearch/common/settings/IndexScopedSettings.java

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -212,6 +212,7 @@ public final class IndexScopedSettings extends AbstractScopedSettings {
212212
IndexSettings.USE_TIME_SERIES_DOC_VALUES_FORMAT_LARGE_BLOCK_SIZE,
213213
InferenceMetadataFieldsMapper.USE_LEGACY_SEMANTIC_TEXT_FORMAT,
214214
IndexSettings.USE_ES_812_POSTINGS_FORMAT,
215+
IndexSettings.USE_DOC_VALUES_SKIPPER,
215216

216217
// validate that built-in similarities don't get redefined
217218
Setting.groupSetting("index.similarity.", (s) -> {
@@ -248,9 +249,6 @@ public final class IndexScopedSettings extends AbstractScopedSettings {
248249
)
249250
);
250251

251-
if (IndexSettings.DOC_VALUES_SKIPPER) {
252-
settings.add(IndexSettings.USE_DOC_VALUES_SKIPPER);
253-
}
254252
if (IndexSettings.TSDB_SYNTHETIC_ID_FEATURE_FLAG) {
255253
settings.add(IndexSettings.USE_SYNTHETIC_ID);
256254
}

server/src/main/java/org/elasticsearch/index/IndexSettings.java

Lines changed: 26 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -770,19 +770,29 @@ public Iterator<Setting<?>> settings() {
770770
Property.ServerlessPublic
771771
);
772772

773-
public static final boolean DOC_VALUES_SKIPPER = new FeatureFlag("doc_values_skipper").isEnabled();
774773
public static final Setting<Boolean> USE_DOC_VALUES_SKIPPER = Setting.boolSetting("index.mapping.use_doc_values_skipper", s -> {
775774
IndexVersion iv = SETTING_INDEX_VERSION_CREATED.get(s);
776775
if (MODE.get(s) == IndexMode.TIME_SERIES) {
776+
if (DiscoveryNode.isStateless(s)) {
777+
if (iv.onOrAfter(IndexVersions.STATELESS_SKIPPERS_ENABLED_FOR_TSDB)) {
778+
return "true";
779+
} else {
780+
return "false";
781+
}
782+
}
777783
if (iv.onOrAfter(IndexVersions.SKIPPERS_ENABLED_BY_DEFAULT)) {
778784
return "true";
779785
}
780786
return "false";
787+
} else {
788+
if (DiscoveryNode.isStateless(s)) {
789+
return "false";
790+
}
791+
if (iv.onOrAfter(IndexVersions.SKIPPERS_ENABLED_BY_DEFAULT) && iv.before(IndexVersions.SKIPPER_DEFAULTS_ONLY_ON_TSDB)) {
792+
return "true";
793+
}
794+
return "false";
781795
}
782-
if (iv.onOrAfter(IndexVersions.SKIPPERS_ENABLED_BY_DEFAULT) && iv.before(IndexVersions.SKIPPER_DEFAULTS_ONLY_ON_TSDB)) {
783-
return "true";
784-
}
785-
return "false";
786796
}, Property.IndexScope, Property.Final);
787797

788798
public static final Setting<SourceFieldMapper.Mode> INDEX_MAPPER_SOURCE_MODE_SETTING = Setting.enumSetting(
@@ -1055,6 +1065,7 @@ private void setRetentionLeaseMillis(final TimeValue retentionLease) {
10551065
private final boolean recoverySourceEnabled;
10561066
private final boolean recoverySourceSyntheticEnabled;
10571067
private final boolean useDocValuesSkipper;
1068+
private final boolean useDocValuesSkipperForHostname;
10581069
private final boolean useTimeSeriesSyntheticId;
10591070
private final boolean useTimeSeriesDocValuesFormat;
10601071
private final boolean useTimeSeriesDocValuesFormatLargeBlockSize;
@@ -1244,7 +1255,10 @@ public IndexSettings(final IndexMetadata indexMetadata, final Settings nodeSetti
12441255
recoverySourceEnabled = RecoverySettings.INDICES_RECOVERY_SOURCE_ENABLED_SETTING.get(nodeSettings);
12451256
recoverySourceSyntheticEnabled = DiscoveryNode.isStateless(nodeSettings) == false
12461257
&& scopedSettings.get(RECOVERY_USE_SYNTHETIC_SOURCE_SETTING);
1247-
useDocValuesSkipper = DOC_VALUES_SKIPPER && scopedSettings.get(USE_DOC_VALUES_SKIPPER);
1258+
useDocValuesSkipper = scopedSettings.get(USE_DOC_VALUES_SKIPPER);
1259+
useDocValuesSkipperForHostname = USE_DOC_VALUES_SKIPPER.exists(settings)
1260+
? scopedSettings.get(USE_DOC_VALUES_SKIPPER)
1261+
: version.onOrAfter(IndexVersions.SKIPPERS_ENABLED_BY_DEFAULT) && version.before(IndexVersions.SKIPPER_DEFAULTS_ONLY_ON_TSDB);
12481262
seqNoIndexOptions = scopedSettings.get(SEQ_NO_INDEX_OPTIONS_SETTING);
12491263
useTimeSeriesDocValuesFormat = scopedSettings.get(USE_TIME_SERIES_DOC_VALUES_FORMAT_SETTING);
12501264
useTimeSeriesDocValuesFormatLargeBlockSize = scopedSettings.get(USE_TIME_SERIES_DOC_VALUES_FORMAT_LARGE_BLOCK_SIZE);
@@ -2012,6 +2026,12 @@ public boolean useDocValuesSkipper() {
20122026
return useDocValuesSkipper;
20132027
}
20142028

2029+
// Necessary because we accidentally made host.name use skippers before the feature flag was
2030+
// removed in serverless
2031+
public boolean useDocValuesSkipperForHostName() {
2032+
return useDocValuesSkipperForHostname;
2033+
}
2034+
20152035
/**
20162036
* @return Whether the index is a time-series index that use synthetic ids.
20172037
*/

server/src/main/java/org/elasticsearch/index/IndexVersions.java

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -206,6 +206,7 @@ private static Version parseUnchecked(String version) {
206206
public static final IndexVersion STORE_IGNORED_KEYWORDS_IN_BINARY_DOC_VALUES = def(9_054_0_00, Version.LUCENE_10_3_2);
207207
public static final IndexVersion TIME_SERIES_USE_STORED_FIELDS_BLOOM_FILTER_FOR_ID = def(9_055_0_00, Version.LUCENE_10_3_2);
208208
public static final IndexVersion AGG_METRIC_DOUBLE_REMOVE_POINTS = def(9_056_0_00, Version.LUCENE_10_3_2);
209+
public static final IndexVersion STATELESS_SKIPPERS_ENABLED_FOR_TSDB = def(9_057_0_00, Version.LUCENE_10_3_2);
209210

210211
/*
211212
* STOP! READ THIS FIRST! No, really,

0 commit comments

Comments
 (0)