[8.x] Remove synthetic recovery source feature flag.

docs/changelog/122615.yaml

@@ -0,0 +1,8 @@
+pr: 122615
+summary: Enable synthetic recovery source by default when synthetic source is enabled.
+  Using synthetic recovery source significantly improves indexing performance compared
+  to regular recovery source.
+area: Mapping
+type: enhancement
+issues:
+ - 116726

qa/smoke-test-multinode/src/yamlRestTest/java/org/elasticsearch/smoketest/SmokeTestMultiNodeClientYamlTestSuiteIT.java

@@ -36,7 +36,6 @@ public class SmokeTestMultiNodeClientYamlTestSuiteIT extends ESClientYamlSuiteTe
         .node(0, n -> n.setting("node.roles", "[master,data,ml,remote_cluster_client,transform]"))
         .feature(FeatureFlag.TIME_SERIES_MODE)
         .feature(FeatureFlag.SUB_OBJECTS_AUTO_ENABLED)
-        .feature(FeatureFlag.INDEX_RECOVERY_USE_SYNTHETIC_SOURCE)
         .build();
 
     public SmokeTestMultiNodeClientYamlTestSuiteIT(@Name("yaml") ClientYamlTestCandidate testCandidate) {

rest-api-spec/src/yamlRestTest/java/org/elasticsearch/test/rest/ClientYamlTestSuiteIT.java

@@ -37,7 +37,6 @@ public class ClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase {
         .module("data-streams")
         .feature(FeatureFlag.TIME_SERIES_MODE)
         .feature(FeatureFlag.SUB_OBJECTS_AUTO_ENABLED)
-        .feature(FeatureFlag.INDEX_RECOVERY_USE_SYNTHETIC_SOURCE)
         .build();
 
     public ClientYamlTestSuiteIT(@Name("yaml") ClientYamlTestCandidate testCandidate) {

server/src/main/java/org/elasticsearch/index/IndexSettings.java

@@ -24,7 +24,6 @@
 import org.elasticsearch.common.time.DateUtils;
 import org.elasticsearch.common.unit.ByteSizeUnit;
 import org.elasticsearch.common.unit.ByteSizeValue;
-import org.elasticsearch.common.util.FeatureFlag;
 import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.features.NodeFeature;
 import org.elasticsearch.index.mapper.IgnoredSourceFieldMapper;
@@ -727,14 +726,11 @@ public Iterator<Setting<?>> settings() {
         Setting.Property.IndexScope
     );
 
-    public static final FeatureFlag RECOVERY_USE_SYNTHETIC_SOURCE = new FeatureFlag("index_recovery_use_synthetic_source");
     public static final Setting<Boolean> RECOVERY_USE_SYNTHETIC_SOURCE_SETTING = Setting.boolSetting(
         "index.recovery.use_synthetic_source",
         settings -> {
-            boolean isSyntheticSourceRecoveryFeatureFlagEnabled = RECOVERY_USE_SYNTHETIC_SOURCE.isEnabled();
-            boolean isNewIndexVersion = SETTING_INDEX_VERSION_CREATED.get(settings)
+            boolean useSyntheticRecoverySource = SETTING_INDEX_VERSION_CREATED.get(settings)
                 .onOrAfter(IndexVersions.USE_SYNTHETIC_SOURCE_FOR_RECOVERY_BY_DEFAULT_BACKPORT);
-            boolean useSyntheticRecoverySource = isSyntheticSourceRecoveryFeatureFlagEnabled && isNewIndexVersion;
             return String.valueOf(
                 useSyntheticRecoverySource
                     && Objects.equals(INDEX_MAPPER_SOURCE_MODE_SETTING.get(settings), SourceFieldMapper.Mode.SYNTHETIC)

server/src/test/java/org/elasticsearch/index/mapper/NativeArrayIntegrationTestCase.java

@@ -20,7 +20,6 @@
 import org.elasticsearch.action.support.WriteRequest;
 import org.elasticsearch.common.network.NetworkAddress;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.index.IndexSettings;
 import org.elasticsearch.index.query.IdsQueryBuilder;
 import org.elasticsearch.test.ESSingleNodeTestCase;
 import org.elasticsearch.xcontent.XContentBuilder;
@@ -35,7 +34,6 @@
 
 import static org.elasticsearch.xcontent.XContentFactory.jsonBuilder;
 import static org.hamcrest.Matchers.contains;
-import static org.hamcrest.Matchers.containsInAnyOrder;
 import static org.hamcrest.Matchers.empty;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.hasKey;
@@ -136,14 +134,7 @@ protected void verifySyntheticArray(Object[][] arrays, XContentBuilder mapping,
                 var document = reader.storedFields().document(i);
                 // Verify that there is no ignored source:
                 Set<String> storedFieldNames = new LinkedHashSet<>(document.getFields().stream().map(IndexableField::name).toList());
-                if (IndexSettings.RECOVERY_USE_SYNTHETIC_SOURCE.isEnabled()) {
-                    assertThat(storedFieldNames, contains(expectedStoredFields));
-                } else {
-                    var copyExpectedStoredFields = new String[expectedStoredFields.length + 1];
-                    System.arraycopy(expectedStoredFields, 0, copyExpectedStoredFields, 0, expectedStoredFields.length);
-                    copyExpectedStoredFields[copyExpectedStoredFields.length - 1] = "_recovery_source";
-                    assertThat(storedFieldNames, containsInAnyOrder(copyExpectedStoredFields));
-                }
+                assertThat(storedFieldNames, contains(expectedStoredFields));
             }
             var fieldInfo = FieldInfos.getMergedFieldInfos(reader).fieldInfo("field.offsets");
             assertThat(fieldInfo.getDocValuesType(), equalTo(DocValuesType.SORTED));
@@ -208,11 +199,7 @@ protected void verifySyntheticObjectArray(List<List<Object[]>> documents) throws
                 var document = reader.storedFields().document(i);
                 // Verify that there is ignored source because of leaf array being wrapped by object array:
                 List<String> storedFieldNames = document.getFields().stream().map(IndexableField::name).toList();
-                if (IndexSettings.RECOVERY_USE_SYNTHETIC_SOURCE.isEnabled()) {
-                    assertThat(storedFieldNames, contains("_id", "_ignored_source"));
-                } else {
-                    assertThat(storedFieldNames, containsInAnyOrder("_id", "_ignored_source", "_recovery_source"));
-                }
+                assertThat(storedFieldNames, contains("_id", "_ignored_source"));
 
                 // Verify that there is no offset field:
                 LeafReader leafReader = reader.leaves().get(0).reader();
@@ -285,11 +272,7 @@ protected void verifySyntheticArrayInObject(List<Object[]> documents) throws IOE
                 var document = reader.storedFields().document(i);
                 // Verify that there is no ignored source:
                 Set<String> storedFieldNames = new LinkedHashSet<>(document.getFields().stream().map(IndexableField::name).toList());
-                if (IndexSettings.RECOVERY_USE_SYNTHETIC_SOURCE.isEnabled()) {
-                    assertThat(storedFieldNames, contains("_id"));
-                } else {
-                    assertThat(storedFieldNames, containsInAnyOrder("_id", "_recovery_source"));
-                }
+                assertThat(storedFieldNames, contains("_id"));
             }
             var fieldInfo = FieldInfos.getMergedFieldInfos(reader).fieldInfo("object.field.offsets");
             assertThat(fieldInfo.getDocValuesType(), equalTo(DocValuesType.SORTED));

server/src/test/java/org/elasticsearch/index/mapper/SourceFieldMapperTests.java

@@ -475,13 +475,7 @@ public void testRecoverySourceWithSyntheticSource() throws IOException {
             MapperService mapperService = createMapperService(settings, topMapping(b -> {}));
             DocumentMapper docMapper = mapperService.documentMapper();
             ParsedDocument doc = docMapper.parse(source(b -> b.field("field1", "value1")));
-            if (IndexSettings.RECOVERY_USE_SYNTHETIC_SOURCE.isEnabled() == false) {
-                // TODO: remove this if branch when removing the 'index_recovery_use_synthetic_source' feature flag
-                assertNotNull(doc.rootDoc().getField("_recovery_source"));
-                assertThat(doc.rootDoc().getField("_recovery_source").binaryValue(), equalTo(new BytesRef("{\"field1\":\"value1\"}")));
-            } else {
-                assertNull(doc.rootDoc().getField("_recovery_source"));
-            }
+            assertNull(doc.rootDoc().getField("_recovery_source"));
         }
         {
             Settings settings = Settings.builder()
@@ -512,16 +506,8 @@ public void testRecoverySourceWithLogs() throws IOException {
             MapperService mapperService = createMapperService(settings, mapping(b -> {}));
             DocumentMapper docMapper = mapperService.documentMapper();
             ParsedDocument doc = docMapper.parse(source(b -> { b.field("@timestamp", "2012-02-13"); }));
-            if (IndexSettings.RECOVERY_USE_SYNTHETIC_SOURCE.isEnabled() == false) {
-                // TODO: remove this if branch when removing the 'index_recovery_use_synthetic_source' feature flag
-                assertNotNull(doc.rootDoc().getField("_recovery_source"));
-                assertThat(
-                    doc.rootDoc().getField("_recovery_source").binaryValue(),
-                    equalTo(new BytesRef("{\"@timestamp\":\"2012-02-13\"}"))
-                );
-            } else {
-                assertNull(doc.rootDoc().getField("_recovery_source"));
-            }
+            assertNotNull(doc.rootDoc().getField("_recovery_source_size"));
+            assertThat(doc.rootDoc().getField("_recovery_source_size").numericValue(), equalTo(27L));
         }
         {
             Settings settings = Settings.builder()
@@ -714,16 +700,7 @@ public void testRecoverySourceWithLogsCustom() throws IOException {
             MapperService mapperService = createMapperService(settings, mappings);
             DocumentMapper docMapper = mapperService.documentMapper();
             ParsedDocument doc = docMapper.parse(source(b -> { b.field("@timestamp", "2012-02-13"); }));
-            if (IndexSettings.RECOVERY_USE_SYNTHETIC_SOURCE.isEnabled() == false) {
-                // TODO: remove this if branch when removing the 'index_recovery_use_synthetic_source' feature flag
-                assertNotNull(doc.rootDoc().getField("_recovery_source"));
-                assertThat(
-                    doc.rootDoc().getField("_recovery_source").binaryValue(),
-                    equalTo(new BytesRef("{\"@timestamp\":\"2012-02-13\"}"))
-                );
-            } else {
-                assertNull(doc.rootDoc().getField("_recovery_source"));
-            }
+            assertNull(doc.rootDoc().getField("_recovery_source"));
         }
         {
             Settings settings = Settings.builder()
@@ -749,16 +726,7 @@ public void testRecoverySourceWithTimeSeries() throws IOException {
             }));
             DocumentMapper docMapper = mapperService.documentMapper();
             ParsedDocument doc = docMapper.parse(source("123", b -> b.field("@timestamp", "2012-02-13").field("field", "value1"), null));
-            if (IndexSettings.RECOVERY_USE_SYNTHETIC_SOURCE.isEnabled() == false) {
-                // TODO: remove this if branch when removing the 'index_recovery_use_synthetic_source' feature flag
-                assertNotNull(doc.rootDoc().getField("_recovery_source"));
-                assertThat(
-                    doc.rootDoc().getField("_recovery_source").binaryValue(),
-                    equalTo(new BytesRef("{\"@timestamp\":\"2012-02-13\",\"field\":\"value1\"}"))
-                );
-            } else {
-                assertNull(doc.rootDoc().getField("_recovery_source"));
-            }
+            assertNull(doc.rootDoc().getField("_recovery_source"));
         }
         {
             Settings settings = Settings.builder()
@@ -802,16 +770,7 @@ public void testRecoverySourceWithTimeSeriesCustom() throws IOException {
             MapperService mapperService = createMapperService(settings, mappings);
             DocumentMapper docMapper = mapperService.documentMapper();
             ParsedDocument doc = docMapper.parse(source("123", b -> b.field("@timestamp", "2012-02-13").field("field", "value1"), null));
-            if (IndexSettings.RECOVERY_USE_SYNTHETIC_SOURCE.isEnabled() == false) {
-                // TODO: remove this if branch when removing the 'index_recovery_use_synthetic_source' feature flag
-                assertNotNull(doc.rootDoc().getField("_recovery_source"));
-                assertThat(
-                    doc.rootDoc().getField("_recovery_source").binaryValue(),
-                    equalTo(new BytesRef("{\"@timestamp\":\"2012-02-13\",\"field\":\"value1\"}"))
-                );
-            } else {
-                assertNull(doc.rootDoc().getField("_recovery_source"));
-            }
+            assertNull(doc.rootDoc().getField("_recovery_source"));
         }
         {
             Settings settings = Settings.builder()

test/test-clusters/src/main/java/org/elasticsearch/test/cluster/FeatureFlag.java

@@ -18,12 +18,7 @@
 public enum FeatureFlag {
     TIME_SERIES_MODE("es.index_mode_feature_flag_registered=true", Version.fromString("8.0.0"), null),
     FAILURE_STORE_ENABLED("es.failure_store_feature_flag_enabled=true", Version.fromString("8.12.0"), null),
-    SUB_OBJECTS_AUTO_ENABLED("es.sub_objects_auto_feature_flag_enabled=true", Version.fromString("8.16.0"), null),
-    INDEX_RECOVERY_USE_SYNTHETIC_SOURCE(
-        "es.index_recovery_use_synthetic_source_feature_flag_enabled=true",
-        Version.fromString("8.18.0"),
-        null
-    );
+    SUB_OBJECTS_AUTO_ENABLED("es.sub_objects_auto_feature_flag_enabled=true", Version.fromString("8.16.0"), null);
 
     public final String systemProperty;
     public final Version from;

x-pack/plugin/logsdb/src/yamlRestTest/java/org/elasticsearch/xpack/logsdb/LogsdbTestSuiteIT.java

@@ -11,7 +11,6 @@
 import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
 
 import org.elasticsearch.test.cluster.ElasticsearchCluster;
-import org.elasticsearch.test.cluster.FeatureFlag;
 import org.elasticsearch.test.cluster.local.distribution.DistributionType;
 import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate;
 import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase;
@@ -24,7 +23,6 @@ public class LogsdbTestSuiteIT extends ESClientYamlSuiteTestCase {
         .distribution(DistributionType.DEFAULT)
         .setting("xpack.security.enabled", "false")
         .setting("xpack.license.self_generated.type", "trial")
-        .feature(FeatureFlag.INDEX_RECOVERY_USE_SYNTHETIC_SOURCE)
         .build();
 
     public LogsdbTestSuiteIT(@Name("yaml") ClientYamlTestCandidate testCandidate) {

x-pack/qa/core-rest-tests-with-security/src/yamlRestTest/java/org/elasticsearch/xpack/security/CoreWithSecurityClientYamlTestSuiteIT.java

@@ -50,7 +50,6 @@ public class CoreWithSecurityClientYamlTestSuiteIT extends ESClientYamlSuiteTest
         .user(USER, PASS)
         .feature(FeatureFlag.TIME_SERIES_MODE)
         .feature(FeatureFlag.SUB_OBJECTS_AUTO_ENABLED)
-        .feature(FeatureFlag.INDEX_RECOVERY_USE_SYNTHETIC_SOURCE)
         .build();
 
     public CoreWithSecurityClientYamlTestSuiteIT(@Name("yaml") ClientYamlTestCandidate testCandidate) {