diff --git a/docs/changelog/122615.yaml b/docs/changelog/122615.yaml new file mode 100644 index 0000000000000..0070fea6a3b95 --- /dev/null +++ b/docs/changelog/122615.yaml @@ -0,0 +1,8 @@ +pr: 122615 +summary: Enable synthetic recovery source by default when synthetic source is enabled. + Using synthetic recovery source significantly improves indexing performance compared + to regular recovery source. +area: Mapping +type: enhancement +issues: + - 116726 diff --git a/qa/smoke-test-multinode/src/yamlRestTest/java/org/elasticsearch/smoketest/SmokeTestMultiNodeClientYamlTestSuiteIT.java b/qa/smoke-test-multinode/src/yamlRestTest/java/org/elasticsearch/smoketest/SmokeTestMultiNodeClientYamlTestSuiteIT.java index 5943d4a86ab92..e53c0564be297 100644 --- a/qa/smoke-test-multinode/src/yamlRestTest/java/org/elasticsearch/smoketest/SmokeTestMultiNodeClientYamlTestSuiteIT.java +++ b/qa/smoke-test-multinode/src/yamlRestTest/java/org/elasticsearch/smoketest/SmokeTestMultiNodeClientYamlTestSuiteIT.java @@ -36,7 +36,6 @@ public class SmokeTestMultiNodeClientYamlTestSuiteIT extends ESClientYamlSuiteTe .node(0, n -> n.setting("node.roles", "[master,data,ml,remote_cluster_client,transform]")) .feature(FeatureFlag.TIME_SERIES_MODE) .feature(FeatureFlag.SUB_OBJECTS_AUTO_ENABLED) - .feature(FeatureFlag.INDEX_RECOVERY_USE_SYNTHETIC_SOURCE) .build(); public SmokeTestMultiNodeClientYamlTestSuiteIT(@Name("yaml") ClientYamlTestCandidate testCandidate) { diff --git a/rest-api-spec/src/yamlRestTest/java/org/elasticsearch/test/rest/ClientYamlTestSuiteIT.java b/rest-api-spec/src/yamlRestTest/java/org/elasticsearch/test/rest/ClientYamlTestSuiteIT.java index 1a70797d8f101..a92f5df53aeec 100644 --- a/rest-api-spec/src/yamlRestTest/java/org/elasticsearch/test/rest/ClientYamlTestSuiteIT.java +++ b/rest-api-spec/src/yamlRestTest/java/org/elasticsearch/test/rest/ClientYamlTestSuiteIT.java @@ -37,7 +37,6 @@ public class ClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase { .module("data-streams") .feature(FeatureFlag.TIME_SERIES_MODE) .feature(FeatureFlag.SUB_OBJECTS_AUTO_ENABLED) - .feature(FeatureFlag.INDEX_RECOVERY_USE_SYNTHETIC_SOURCE) .build(); public ClientYamlTestSuiteIT(@Name("yaml") ClientYamlTestCandidate testCandidate) { diff --git a/server/src/main/java/org/elasticsearch/index/IndexSettings.java b/server/src/main/java/org/elasticsearch/index/IndexSettings.java index 9961175086a96..2707e0ccf8041 100644 --- a/server/src/main/java/org/elasticsearch/index/IndexSettings.java +++ b/server/src/main/java/org/elasticsearch/index/IndexSettings.java @@ -24,7 +24,6 @@ import org.elasticsearch.common.time.DateUtils; import org.elasticsearch.common.unit.ByteSizeUnit; import org.elasticsearch.common.unit.ByteSizeValue; -import org.elasticsearch.common.util.FeatureFlag; import org.elasticsearch.core.TimeValue; import org.elasticsearch.features.NodeFeature; import org.elasticsearch.index.mapper.IgnoredSourceFieldMapper; @@ -727,14 +726,11 @@ public Iterator> settings() { Setting.Property.IndexScope ); - public static final FeatureFlag RECOVERY_USE_SYNTHETIC_SOURCE = new FeatureFlag("index_recovery_use_synthetic_source"); public static final Setting RECOVERY_USE_SYNTHETIC_SOURCE_SETTING = Setting.boolSetting( "index.recovery.use_synthetic_source", settings -> { - boolean isSyntheticSourceRecoveryFeatureFlagEnabled = RECOVERY_USE_SYNTHETIC_SOURCE.isEnabled(); - boolean isNewIndexVersion = SETTING_INDEX_VERSION_CREATED.get(settings) + boolean useSyntheticRecoverySource = SETTING_INDEX_VERSION_CREATED.get(settings) .onOrAfter(IndexVersions.USE_SYNTHETIC_SOURCE_FOR_RECOVERY_BY_DEFAULT_BACKPORT); - boolean useSyntheticRecoverySource = isSyntheticSourceRecoveryFeatureFlagEnabled && isNewIndexVersion; return String.valueOf( useSyntheticRecoverySource && Objects.equals(INDEX_MAPPER_SOURCE_MODE_SETTING.get(settings), SourceFieldMapper.Mode.SYNTHETIC) diff --git a/server/src/test/java/org/elasticsearch/index/mapper/NativeArrayIntegrationTestCase.java b/server/src/test/java/org/elasticsearch/index/mapper/NativeArrayIntegrationTestCase.java index 7283dc822e12f..4b44f2444f27e 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/NativeArrayIntegrationTestCase.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/NativeArrayIntegrationTestCase.java @@ -20,7 +20,6 @@ import org.elasticsearch.action.support.WriteRequest; import org.elasticsearch.common.network.NetworkAddress; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.query.IdsQueryBuilder; import org.elasticsearch.test.ESSingleNodeTestCase; import org.elasticsearch.xcontent.XContentBuilder; @@ -35,7 +34,6 @@ import static org.elasticsearch.xcontent.XContentFactory.jsonBuilder; import static org.hamcrest.Matchers.contains; -import static org.hamcrest.Matchers.containsInAnyOrder; import static org.hamcrest.Matchers.empty; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.hasKey; @@ -136,14 +134,7 @@ protected void verifySyntheticArray(Object[][] arrays, XContentBuilder mapping, var document = reader.storedFields().document(i); // Verify that there is no ignored source: Set storedFieldNames = new LinkedHashSet<>(document.getFields().stream().map(IndexableField::name).toList()); - if (IndexSettings.RECOVERY_USE_SYNTHETIC_SOURCE.isEnabled()) { - assertThat(storedFieldNames, contains(expectedStoredFields)); - } else { - var copyExpectedStoredFields = new String[expectedStoredFields.length + 1]; - System.arraycopy(expectedStoredFields, 0, copyExpectedStoredFields, 0, expectedStoredFields.length); - copyExpectedStoredFields[copyExpectedStoredFields.length - 1] = "_recovery_source"; - assertThat(storedFieldNames, containsInAnyOrder(copyExpectedStoredFields)); - } + assertThat(storedFieldNames, contains(expectedStoredFields)); } var fieldInfo = FieldInfos.getMergedFieldInfos(reader).fieldInfo("field.offsets"); assertThat(fieldInfo.getDocValuesType(), equalTo(DocValuesType.SORTED)); @@ -208,11 +199,7 @@ protected void verifySyntheticObjectArray(List> documents) throws var document = reader.storedFields().document(i); // Verify that there is ignored source because of leaf array being wrapped by object array: List storedFieldNames = document.getFields().stream().map(IndexableField::name).toList(); - if (IndexSettings.RECOVERY_USE_SYNTHETIC_SOURCE.isEnabled()) { - assertThat(storedFieldNames, contains("_id", "_ignored_source")); - } else { - assertThat(storedFieldNames, containsInAnyOrder("_id", "_ignored_source", "_recovery_source")); - } + assertThat(storedFieldNames, contains("_id", "_ignored_source")); // Verify that there is no offset field: LeafReader leafReader = reader.leaves().get(0).reader(); @@ -285,11 +272,7 @@ protected void verifySyntheticArrayInObject(List documents) throws IOE var document = reader.storedFields().document(i); // Verify that there is no ignored source: Set storedFieldNames = new LinkedHashSet<>(document.getFields().stream().map(IndexableField::name).toList()); - if (IndexSettings.RECOVERY_USE_SYNTHETIC_SOURCE.isEnabled()) { - assertThat(storedFieldNames, contains("_id")); - } else { - assertThat(storedFieldNames, containsInAnyOrder("_id", "_recovery_source")); - } + assertThat(storedFieldNames, contains("_id")); } var fieldInfo = FieldInfos.getMergedFieldInfos(reader).fieldInfo("object.field.offsets"); assertThat(fieldInfo.getDocValuesType(), equalTo(DocValuesType.SORTED)); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/SourceFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/SourceFieldMapperTests.java index f33cda7d77170..d6f17bbbec24e 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/SourceFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/SourceFieldMapperTests.java @@ -475,13 +475,7 @@ public void testRecoverySourceWithSyntheticSource() throws IOException { MapperService mapperService = createMapperService(settings, topMapping(b -> {})); DocumentMapper docMapper = mapperService.documentMapper(); ParsedDocument doc = docMapper.parse(source(b -> b.field("field1", "value1"))); - if (IndexSettings.RECOVERY_USE_SYNTHETIC_SOURCE.isEnabled() == false) { - // TODO: remove this if branch when removing the 'index_recovery_use_synthetic_source' feature flag - assertNotNull(doc.rootDoc().getField("_recovery_source")); - assertThat(doc.rootDoc().getField("_recovery_source").binaryValue(), equalTo(new BytesRef("{\"field1\":\"value1\"}"))); - } else { - assertNull(doc.rootDoc().getField("_recovery_source")); - } + assertNull(doc.rootDoc().getField("_recovery_source")); } { Settings settings = Settings.builder() @@ -512,16 +506,8 @@ public void testRecoverySourceWithLogs() throws IOException { MapperService mapperService = createMapperService(settings, mapping(b -> {})); DocumentMapper docMapper = mapperService.documentMapper(); ParsedDocument doc = docMapper.parse(source(b -> { b.field("@timestamp", "2012-02-13"); })); - if (IndexSettings.RECOVERY_USE_SYNTHETIC_SOURCE.isEnabled() == false) { - // TODO: remove this if branch when removing the 'index_recovery_use_synthetic_source' feature flag - assertNotNull(doc.rootDoc().getField("_recovery_source")); - assertThat( - doc.rootDoc().getField("_recovery_source").binaryValue(), - equalTo(new BytesRef("{\"@timestamp\":\"2012-02-13\"}")) - ); - } else { - assertNull(doc.rootDoc().getField("_recovery_source")); - } + assertNotNull(doc.rootDoc().getField("_recovery_source_size")); + assertThat(doc.rootDoc().getField("_recovery_source_size").numericValue(), equalTo(27L)); } { Settings settings = Settings.builder() @@ -714,16 +700,7 @@ public void testRecoverySourceWithLogsCustom() throws IOException { MapperService mapperService = createMapperService(settings, mappings); DocumentMapper docMapper = mapperService.documentMapper(); ParsedDocument doc = docMapper.parse(source(b -> { b.field("@timestamp", "2012-02-13"); })); - if (IndexSettings.RECOVERY_USE_SYNTHETIC_SOURCE.isEnabled() == false) { - // TODO: remove this if branch when removing the 'index_recovery_use_synthetic_source' feature flag - assertNotNull(doc.rootDoc().getField("_recovery_source")); - assertThat( - doc.rootDoc().getField("_recovery_source").binaryValue(), - equalTo(new BytesRef("{\"@timestamp\":\"2012-02-13\"}")) - ); - } else { - assertNull(doc.rootDoc().getField("_recovery_source")); - } + assertNull(doc.rootDoc().getField("_recovery_source")); } { Settings settings = Settings.builder() @@ -749,16 +726,7 @@ public void testRecoverySourceWithTimeSeries() throws IOException { })); DocumentMapper docMapper = mapperService.documentMapper(); ParsedDocument doc = docMapper.parse(source("123", b -> b.field("@timestamp", "2012-02-13").field("field", "value1"), null)); - if (IndexSettings.RECOVERY_USE_SYNTHETIC_SOURCE.isEnabled() == false) { - // TODO: remove this if branch when removing the 'index_recovery_use_synthetic_source' feature flag - assertNotNull(doc.rootDoc().getField("_recovery_source")); - assertThat( - doc.rootDoc().getField("_recovery_source").binaryValue(), - equalTo(new BytesRef("{\"@timestamp\":\"2012-02-13\",\"field\":\"value1\"}")) - ); - } else { - assertNull(doc.rootDoc().getField("_recovery_source")); - } + assertNull(doc.rootDoc().getField("_recovery_source")); } { Settings settings = Settings.builder() @@ -802,16 +770,7 @@ public void testRecoverySourceWithTimeSeriesCustom() throws IOException { MapperService mapperService = createMapperService(settings, mappings); DocumentMapper docMapper = mapperService.documentMapper(); ParsedDocument doc = docMapper.parse(source("123", b -> b.field("@timestamp", "2012-02-13").field("field", "value1"), null)); - if (IndexSettings.RECOVERY_USE_SYNTHETIC_SOURCE.isEnabled() == false) { - // TODO: remove this if branch when removing the 'index_recovery_use_synthetic_source' feature flag - assertNotNull(doc.rootDoc().getField("_recovery_source")); - assertThat( - doc.rootDoc().getField("_recovery_source").binaryValue(), - equalTo(new BytesRef("{\"@timestamp\":\"2012-02-13\",\"field\":\"value1\"}")) - ); - } else { - assertNull(doc.rootDoc().getField("_recovery_source")); - } + assertNull(doc.rootDoc().getField("_recovery_source")); } { Settings settings = Settings.builder() diff --git a/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/FeatureFlag.java b/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/FeatureFlag.java index 013a502d93cef..11787866af0d7 100644 --- a/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/FeatureFlag.java +++ b/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/FeatureFlag.java @@ -18,12 +18,7 @@ public enum FeatureFlag { TIME_SERIES_MODE("es.index_mode_feature_flag_registered=true", Version.fromString("8.0.0"), null), FAILURE_STORE_ENABLED("es.failure_store_feature_flag_enabled=true", Version.fromString("8.12.0"), null), - SUB_OBJECTS_AUTO_ENABLED("es.sub_objects_auto_feature_flag_enabled=true", Version.fromString("8.16.0"), null), - INDEX_RECOVERY_USE_SYNTHETIC_SOURCE( - "es.index_recovery_use_synthetic_source_feature_flag_enabled=true", - Version.fromString("8.18.0"), - null - ); + SUB_OBJECTS_AUTO_ENABLED("es.sub_objects_auto_feature_flag_enabled=true", Version.fromString("8.16.0"), null); public final String systemProperty; public final Version from; diff --git a/x-pack/plugin/logsdb/src/yamlRestTest/java/org/elasticsearch/xpack/logsdb/LogsdbTestSuiteIT.java b/x-pack/plugin/logsdb/src/yamlRestTest/java/org/elasticsearch/xpack/logsdb/LogsdbTestSuiteIT.java index a683305c9fde3..fcac791cb2057 100644 --- a/x-pack/plugin/logsdb/src/yamlRestTest/java/org/elasticsearch/xpack/logsdb/LogsdbTestSuiteIT.java +++ b/x-pack/plugin/logsdb/src/yamlRestTest/java/org/elasticsearch/xpack/logsdb/LogsdbTestSuiteIT.java @@ -11,7 +11,6 @@ import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; import org.elasticsearch.test.cluster.ElasticsearchCluster; -import org.elasticsearch.test.cluster.FeatureFlag; import org.elasticsearch.test.cluster.local.distribution.DistributionType; import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate; import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase; @@ -24,7 +23,6 @@ public class LogsdbTestSuiteIT extends ESClientYamlSuiteTestCase { .distribution(DistributionType.DEFAULT) .setting("xpack.security.enabled", "false") .setting("xpack.license.self_generated.type", "trial") - .feature(FeatureFlag.INDEX_RECOVERY_USE_SYNTHETIC_SOURCE) .build(); public LogsdbTestSuiteIT(@Name("yaml") ClientYamlTestCandidate testCandidate) { diff --git a/x-pack/qa/core-rest-tests-with-security/src/yamlRestTest/java/org/elasticsearch/xpack/security/CoreWithSecurityClientYamlTestSuiteIT.java b/x-pack/qa/core-rest-tests-with-security/src/yamlRestTest/java/org/elasticsearch/xpack/security/CoreWithSecurityClientYamlTestSuiteIT.java index 16baa729d9703..94eeee5ed9298 100644 --- a/x-pack/qa/core-rest-tests-with-security/src/yamlRestTest/java/org/elasticsearch/xpack/security/CoreWithSecurityClientYamlTestSuiteIT.java +++ b/x-pack/qa/core-rest-tests-with-security/src/yamlRestTest/java/org/elasticsearch/xpack/security/CoreWithSecurityClientYamlTestSuiteIT.java @@ -50,7 +50,6 @@ public class CoreWithSecurityClientYamlTestSuiteIT extends ESClientYamlSuiteTest .user(USER, PASS) .feature(FeatureFlag.TIME_SERIES_MODE) .feature(FeatureFlag.SUB_OBJECTS_AUTO_ENABLED) - .feature(FeatureFlag.INDEX_RECOVERY_USE_SYNTHETIC_SOURCE) .build(); public CoreWithSecurityClientYamlTestSuiteIT(@Name("yaml") ClientYamlTestCandidate testCandidate) {