From 2e7c5b395906dee762354ccc801c5a11a2fd6500 Mon Sep 17 00:00:00 2001 From: Salvatore Campagna Date: Thu, 19 Dec 2024 16:53:18 +0100 Subject: [PATCH 01/54] feature: use synthetic recovery source by default for tsdb and logsdb --- .../org/elasticsearch/common/settings/Setting.java | 9 +++++++++ .../java/org/elasticsearch/index/IndexSettings.java | 11 ++++++++++- 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/server/src/main/java/org/elasticsearch/common/settings/Setting.java b/server/src/main/java/org/elasticsearch/common/settings/Setting.java index aec9c108d898d..ad8e6052e1210 100644 --- a/server/src/main/java/org/elasticsearch/common/settings/Setting.java +++ b/server/src/main/java/org/elasticsearch/common/settings/Setting.java @@ -1569,6 +1569,15 @@ public static Setting boolSetting(String key, boolean defaultValue, Val return new Setting<>(key, Boolean.toString(defaultValue), booleanParser(key, properties), validator, properties); } + public static Setting boolSetting( + String key, + Function defaultValueFn, + Validator validator, + Property... properties + ) { + return new Setting<>(key, defaultValueFn, booleanParser(key, properties), validator, properties); + } + public static Setting boolSetting(String key, Function defaultValueFn, Property... properties) { return new Setting<>(key, defaultValueFn, booleanParser(key, properties), properties); } diff --git a/server/src/main/java/org/elasticsearch/index/IndexSettings.java b/server/src/main/java/org/elasticsearch/index/IndexSettings.java index c8a983a48ff16..acc95ffe1c0ab 100644 --- a/server/src/main/java/org/elasticsearch/index/IndexSettings.java +++ b/server/src/main/java/org/elasticsearch/index/IndexSettings.java @@ -657,7 +657,16 @@ public Iterator> settings() { public static final Setting RECOVERY_USE_SYNTHETIC_SOURCE_SETTING = Setting.boolSetting( "index.recovery.use_synthetic_source", - false, + settings -> { + if (IndexSettings.MODE.get(settings) == IndexMode.LOGSDB + || IndexSettings.MODE.get(settings) == IndexMode.TIME_SERIES + && IndexMetadata.SETTING_INDEX_VERSION_CREATED.get(settings) + .onOrAfter(IndexVersions.USE_SYNTHETIC_SOURCE_FOR_RECOVERY)) { + return String.valueOf(true); + } else { + return String.valueOf(false); + } + }, new Setting.Validator<>() { @Override public void validate(Boolean value) {} From 116bab30d26e0604e26260df12dd43f03aa8aafc Mon Sep 17 00:00:00 2001 From: Salvatore Campagna Date: Wed, 8 Jan 2025 10:52:10 +0100 Subject: [PATCH 02/54] fix: default synthetic recovery source value --- .../main/java/org/elasticsearch/index/IndexSettings.java | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/index/IndexSettings.java b/server/src/main/java/org/elasticsearch/index/IndexSettings.java index acc95ffe1c0ab..31cf524d2272f 100644 --- a/server/src/main/java/org/elasticsearch/index/IndexSettings.java +++ b/server/src/main/java/org/elasticsearch/index/IndexSettings.java @@ -658,14 +658,11 @@ public Iterator> settings() { public static final Setting RECOVERY_USE_SYNTHETIC_SOURCE_SETTING = Setting.boolSetting( "index.recovery.use_synthetic_source", settings -> { - if (IndexSettings.MODE.get(settings) == IndexMode.LOGSDB - || IndexSettings.MODE.get(settings) == IndexMode.TIME_SERIES - && IndexMetadata.SETTING_INDEX_VERSION_CREATED.get(settings) - .onOrAfter(IndexVersions.USE_SYNTHETIC_SOURCE_FOR_RECOVERY)) { - return String.valueOf(true); - } else { + if (INDEX_MAPPER_SOURCE_MODE_SETTING.exists(settings) == false) { return String.valueOf(false); } + final SourceFieldMapper.Mode sourceMode = INDEX_MAPPER_SOURCE_MODE_SETTING.get(settings); + return String.valueOf(sourceMode == SourceFieldMapper.Mode.SYNTHETIC); }, new Setting.Validator<>() { @Override From 5c3507ca10ff9ef496c62ed4c7b2a0a5ac77d88c Mon Sep 17 00:00:00 2001 From: Salvatore Campagna Date: Wed, 8 Jan 2025 11:11:37 +0100 Subject: [PATCH 03/54] fix: handle null pointer exception --- .../java/org/elasticsearch/index/mapper/SourceFieldMapper.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/server/src/main/java/org/elasticsearch/index/mapper/SourceFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/SourceFieldMapper.java index f2823ab06fe36..90709d373fc3c 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/SourceFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/SourceFieldMapper.java @@ -69,6 +69,9 @@ public class SourceFieldMapper extends MetadataFieldMapper { public static final Setting INDEX_MAPPER_SOURCE_MODE_SETTING = Setting.enumSetting(SourceFieldMapper.Mode.class, settings -> { final IndexMode indexMode = IndexSettings.MODE.get(settings); + if (indexMode == null) { + return Mode.DISABLED.name(); + } return indexMode.defaultSourceMode().name(); }, "index.mapping.source.mode", value -> {}, Setting.Property.Final, Setting.Property.IndexScope); From 1f4bfed87d583f7b1b8d5c081b856f5bdf0f7acb Mon Sep 17 00:00:00 2001 From: Salvatore Campagna Date: Wed, 8 Jan 2025 13:53:29 +0100 Subject: [PATCH 04/54] fix: depend on mode setting --- .../index/mapper/SourceFieldMapper.java | 21 ++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/index/mapper/SourceFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/SourceFieldMapper.java index 90709d373fc3c..d448fe81d2b4c 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/SourceFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/SourceFieldMapper.java @@ -37,9 +37,12 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; +import java.util.Iterator; import java.util.List; import java.util.Locale; +import static org.elasticsearch.index.IndexSettings.MODE; + public class SourceFieldMapper extends MetadataFieldMapper { public static final NodeFeature SYNTHETIC_SOURCE_FALLBACK = new NodeFeature("mapper.source.synthetic_source_fallback"); public static final NodeFeature SYNTHETIC_SOURCE_STORED_FIELDS_ADVANCE_FIX = new NodeFeature( @@ -68,12 +71,20 @@ public class SourceFieldMapper extends MetadataFieldMapper { public static final String LOSSY_PARAMETERS_ALLOWED_SETTING_NAME = "index.lossy.source-mapping-parameters"; public static final Setting INDEX_MAPPER_SOURCE_MODE_SETTING = Setting.enumSetting(SourceFieldMapper.Mode.class, settings -> { - final IndexMode indexMode = IndexSettings.MODE.get(settings); - if (indexMode == null) { - return Mode.DISABLED.name(); - } + final IndexMode indexMode = MODE.get(settings); return indexMode.defaultSourceMode().name(); - }, "index.mapping.source.mode", value -> {}, Setting.Property.Final, Setting.Property.IndexScope); + }, "index.mapping.source.mode", new Setting.Validator<>() { + @Override + public void validate(Mode value) { + + } + + @Override + public Iterator> settings() { + List> res = List.of(MODE); + return res.iterator(); + } + }, Setting.Property.Final, Setting.Property.IndexScope); public static final String DEPRECATION_WARNING = "Configuring source mode in mappings is deprecated and will be removed " + "in future versions. Use [index.mapping.source.mode] index setting instead."; From 3313218b88731048bcfaf9fa360981f8fe2db942 Mon Sep 17 00:00:00 2001 From: Salvatore Campagna Date: Wed, 8 Jan 2025 13:59:33 +0100 Subject: [PATCH 05/54] fix: restore original code --- .../java/org/elasticsearch/index/mapper/SourceFieldMapper.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/main/java/org/elasticsearch/index/mapper/SourceFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/SourceFieldMapper.java index d448fe81d2b4c..3efd43668face 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/SourceFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/SourceFieldMapper.java @@ -71,7 +71,7 @@ public class SourceFieldMapper extends MetadataFieldMapper { public static final String LOSSY_PARAMETERS_ALLOWED_SETTING_NAME = "index.lossy.source-mapping-parameters"; public static final Setting INDEX_MAPPER_SOURCE_MODE_SETTING = Setting.enumSetting(SourceFieldMapper.Mode.class, settings -> { - final IndexMode indexMode = MODE.get(settings); + final IndexMode indexMode = IndexSettings.MODE.get(settings); return indexMode.defaultSourceMode().name(); }, "index.mapping.source.mode", new Setting.Validator<>() { @Override From 6de37a87538aeaf82046e5a8404e9019ff48568a Mon Sep 17 00:00:00 2001 From: Salvatore Campagna Date: Wed, 8 Jan 2025 14:01:06 +0100 Subject: [PATCH 06/54] nit: remove import MODE --- .../org/elasticsearch/index/mapper/SourceFieldMapper.java | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/index/mapper/SourceFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/SourceFieldMapper.java index 3efd43668face..4c29e81e380f2 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/SourceFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/SourceFieldMapper.java @@ -41,8 +41,6 @@ import java.util.List; import java.util.Locale; -import static org.elasticsearch.index.IndexSettings.MODE; - public class SourceFieldMapper extends MetadataFieldMapper { public static final NodeFeature SYNTHETIC_SOURCE_FALLBACK = new NodeFeature("mapper.source.synthetic_source_fallback"); public static final NodeFeature SYNTHETIC_SOURCE_STORED_FIELDS_ADVANCE_FIX = new NodeFeature( @@ -81,7 +79,7 @@ public void validate(Mode value) { @Override public Iterator> settings() { - List> res = List.of(MODE); + List> res = List.of(IndexSettings.MODE); return res.iterator(); } }, Setting.Property.Final, Setting.Property.IndexScope); From a7ba43aa07411be0bbd5a3aed11a2993daf04af4 Mon Sep 17 00:00:00 2001 From: Salvatore Campagna Date: Wed, 8 Jan 2025 14:02:29 +0100 Subject: [PATCH 07/54] fix: remove unnecessary setting existence check --- .../src/main/java/org/elasticsearch/index/IndexSettings.java | 3 --- 1 file changed, 3 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/index/IndexSettings.java b/server/src/main/java/org/elasticsearch/index/IndexSettings.java index 31cf524d2272f..00ab2bbf7847b 100644 --- a/server/src/main/java/org/elasticsearch/index/IndexSettings.java +++ b/server/src/main/java/org/elasticsearch/index/IndexSettings.java @@ -658,9 +658,6 @@ public Iterator> settings() { public static final Setting RECOVERY_USE_SYNTHETIC_SOURCE_SETTING = Setting.boolSetting( "index.recovery.use_synthetic_source", settings -> { - if (INDEX_MAPPER_SOURCE_MODE_SETTING.exists(settings) == false) { - return String.valueOf(false); - } final SourceFieldMapper.Mode sourceMode = INDEX_MAPPER_SOURCE_MODE_SETTING.get(settings); return String.valueOf(sourceMode == SourceFieldMapper.Mode.SYNTHETIC); }, From f6613d38b9817f3eea637ab119a6902e2ceda873 Mon Sep 17 00:00:00 2001 From: Salvatore Campagna Date: Wed, 8 Jan 2025 14:30:59 +0100 Subject: [PATCH 08/54] fix: move source mode setting to IndexSettings --- .../admin/cluster/stats/MappingStats.java | 4 +- .../metadata/MetadataCreateIndexService.java | 3 +- .../common/settings/IndexScopedSettings.java | 3 +- .../elasticsearch/index/IndexSettings.java | 93 +++++++++++-------- .../index/mapper/SourceFieldMapper.java | 28 +----- .../snapshots/RestoreService.java | 3 +- .../cluster/stats/MappingStatsTests.java | 8 +- ...neSyntheticSourceChangesSnapshotTests.java | 4 +- .../TranslogOperationAsserterTests.java | 8 +- ...edSourceFieldMapperConfigurationTests.java | 3 +- .../index/mapper/SourceFieldMapperTests.java | 30 +++--- .../index/mapper/MapperTestCase.java | 2 +- .../xpack/ccr/FollowIndexIT.java | 2 +- .../TransportResumeFollowActionTests.java | 3 +- .../SourceOnlySnapshotShardTests.java | 4 +- .../logsdb/LogsDBUsageTransportAction.java | 5 +- .../LogsdbIndexModeSettingsProvider.java | 6 +- .../logsdb/LegacyLicenceIntegrationTests.java | 6 +- .../LogsdbIndexModeSettingsProviderTests.java | 10 +- ...dexSettingsProviderLegacyLicenseTests.java | 16 ++-- 20 files changed, 113 insertions(+), 128 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/MappingStats.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/MappingStats.java index 1bc2e1d13c864..29a124b3d0b20 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/MappingStats.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/MappingStats.java @@ -21,7 +21,7 @@ import org.elasticsearch.common.unit.ByteSizeValue; import org.elasticsearch.core.Nullable; import org.elasticsearch.features.NodeFeature; -import org.elasticsearch.index.mapper.SourceFieldMapper; +import org.elasticsearch.index.IndexSettings; import org.elasticsearch.xcontent.ToXContentFragment; import org.elasticsearch.xcontent.XContentBuilder; @@ -71,7 +71,7 @@ public static MappingStats of(Metadata metadata, Runnable ensureNotCancelled) { } AnalysisStats.countMapping(mappingCounts, indexMetadata); - var sourceMode = SourceFieldMapper.INDEX_MAPPER_SOURCE_MODE_SETTING.get(indexMetadata.getSettings()); + var sourceMode = IndexSettings.INDEX_MAPPER_SOURCE_MODE_SETTING.get(indexMetadata.getSettings()); sourceModeUsageCount.merge(sourceMode.toString().toLowerCase(Locale.ENGLISH), 1, Integer::sum); } final AtomicLong totalFieldCount = new AtomicLong(); diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexService.java index a99822c9de00c..83988737e0425 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexService.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexService.java @@ -69,7 +69,6 @@ import org.elasticsearch.index.mapper.DocumentMapper; import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.mapper.MapperService.MergeReason; -import org.elasticsearch.index.mapper.SourceFieldMapper; import org.elasticsearch.index.query.SearchExecutionContext; import org.elasticsearch.index.shard.IndexLongFieldRange; import org.elasticsearch.indices.IndexCreationException; @@ -1590,7 +1589,7 @@ static void validateCloneIndex( private static final Set UNMODIFIABLE_SETTINGS_DURING_RESIZE = Set.of( IndexSettings.MODE.getKey(), - SourceFieldMapper.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), + IndexSettings.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), IndexSettings.RECOVERY_USE_SYNTHETIC_SOURCE_SETTING.getKey(), IndexSortConfig.INDEX_SORT_FIELD_SETTING.getKey(), IndexSortConfig.INDEX_SORT_ORDER_SETTING.getKey(), diff --git a/server/src/main/java/org/elasticsearch/common/settings/IndexScopedSettings.java b/server/src/main/java/org/elasticsearch/common/settings/IndexScopedSettings.java index 48099edeffdaf..a40ab10800bdf 100644 --- a/server/src/main/java/org/elasticsearch/common/settings/IndexScopedSettings.java +++ b/server/src/main/java/org/elasticsearch/common/settings/IndexScopedSettings.java @@ -36,7 +36,6 @@ import org.elasticsearch.index.mapper.IgnoredSourceFieldMapper; import org.elasticsearch.index.mapper.InferenceMetadataFieldsMapper; import org.elasticsearch.index.mapper.MapperService; -import org.elasticsearch.index.mapper.SourceFieldMapper; import org.elasticsearch.index.similarity.SimilarityService; import org.elasticsearch.index.store.FsDirectoryFactory; import org.elasticsearch.index.store.Store; @@ -189,7 +188,7 @@ public final class IndexScopedSettings extends AbstractScopedSettings { FieldMapper.SYNTHETIC_SOURCE_KEEP_INDEX_SETTING, IgnoredSourceFieldMapper.SKIP_IGNORED_SOURCE_WRITE_SETTING, IgnoredSourceFieldMapper.SKIP_IGNORED_SOURCE_READ_SETTING, - SourceFieldMapper.INDEX_MAPPER_SOURCE_MODE_SETTING, + IndexSettings.INDEX_MAPPER_SOURCE_MODE_SETTING, IndexSettings.RECOVERY_USE_SYNTHETIC_SOURCE_SETTING, InferenceMetadataFieldsMapper.USE_LEGACY_SEMANTIC_TEXT_FORMAT, diff --git a/server/src/main/java/org/elasticsearch/index/IndexSettings.java b/server/src/main/java/org/elasticsearch/index/IndexSettings.java index 00ab2bbf7847b..6b13f86fabe8e 100644 --- a/server/src/main/java/org/elasticsearch/index/IndexSettings.java +++ b/server/src/main/java/org/elasticsearch/index/IndexSettings.java @@ -52,7 +52,6 @@ import static org.elasticsearch.index.mapper.MapperService.INDEX_MAPPING_NESTED_DOCS_LIMIT_SETTING; import static org.elasticsearch.index.mapper.MapperService.INDEX_MAPPING_NESTED_FIELDS_LIMIT_SETTING; import static org.elasticsearch.index.mapper.MapperService.INDEX_MAPPING_TOTAL_FIELDS_LIMIT_SETTING; -import static org.elasticsearch.index.mapper.SourceFieldMapper.INDEX_MAPPER_SOURCE_MODE_SETTING; /** * This class encapsulates all index level settings and handles settings updates. @@ -655,6 +654,58 @@ public Iterator> settings() { Property.Final ); + /** + * Returns true if TSDB encoding is enabled. The default is true + */ + public boolean isES87TSDBCodecEnabled() { + return es87TSDBCodecEnabled; + } + + public static final Setting LOGSDB_ROUTE_ON_SORT_FIELDS = Setting.boolSetting( + "index.logsdb.route_on_sort_fields", + false, + Property.IndexScope, + Property.Final + ); + + /** + * The {@link IndexMode "mode"} of the index. + */ + public static final Setting MODE = Setting.enumSetting( + IndexMode.class, + "index.mode", + IndexMode.STANDARD, + new Setting.Validator<>() { + @Override + public void validate(IndexMode value) {} + + @Override + public void validate(IndexMode value, Map, Object> settings) { + value.validateWithOtherSettings(settings); + } + + @Override + public Iterator> settings() { + return IndexMode.VALIDATE_WITH_SETTINGS.iterator(); + } + }, + Property.IndexScope, + Property.Final, + Property.ServerlessPublic + ); + + public static final Setting INDEX_MAPPER_SOURCE_MODE_SETTING = Setting.enumSetting( + SourceFieldMapper.Mode.class, + settings -> { + final IndexMode indexMode = IndexSettings.MODE.get(settings); + return indexMode.defaultSourceMode().name(); + }, + "index.mapping.source.mode", + value -> {}, + Setting.Property.Final, + Setting.Property.IndexScope + ); + public static final Setting RECOVERY_USE_SYNTHETIC_SOURCE_SETTING = Setting.boolSetting( "index.recovery.use_synthetic_source", settings -> { @@ -700,46 +751,6 @@ public Iterator> settings() { Property.Final ); - /** - * Returns true if TSDB encoding is enabled. The default is true - */ - public boolean isES87TSDBCodecEnabled() { - return es87TSDBCodecEnabled; - } - - public static final Setting LOGSDB_ROUTE_ON_SORT_FIELDS = Setting.boolSetting( - "index.logsdb.route_on_sort_fields", - false, - Property.IndexScope, - Property.Final - ); - - /** - * The {@link IndexMode "mode"} of the index. - */ - public static final Setting MODE = Setting.enumSetting( - IndexMode.class, - "index.mode", - IndexMode.STANDARD, - new Setting.Validator<>() { - @Override - public void validate(IndexMode value) {} - - @Override - public void validate(IndexMode value, Map, Object> settings) { - value.validateWithOtherSettings(settings); - } - - @Override - public Iterator> settings() { - return IndexMode.VALIDATE_WITH_SETTINGS.iterator(); - } - }, - Property.IndexScope, - Property.Final, - Property.ServerlessPublic - ); - /** * Legacy index setting, kept for 7.x BWC compatibility. This setting has no effect in 8.x. Do not use. * TODO: Remove in 9.0 diff --git a/server/src/main/java/org/elasticsearch/index/mapper/SourceFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/SourceFieldMapper.java index 4c29e81e380f2..8b7d2ea1d4e75 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/SourceFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/SourceFieldMapper.java @@ -18,7 +18,6 @@ import org.elasticsearch.common.Explicit; import org.elasticsearch.common.Strings; import org.elasticsearch.common.bytes.BytesReference; -import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.CollectionUtils; import org.elasticsearch.core.Nullable; @@ -37,7 +36,6 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; -import java.util.Iterator; import java.util.List; import java.util.Locale; @@ -68,22 +66,6 @@ public class SourceFieldMapper extends MetadataFieldMapper { public static final String LOSSY_PARAMETERS_ALLOWED_SETTING_NAME = "index.lossy.source-mapping-parameters"; - public static final Setting INDEX_MAPPER_SOURCE_MODE_SETTING = Setting.enumSetting(SourceFieldMapper.Mode.class, settings -> { - final IndexMode indexMode = IndexSettings.MODE.get(settings); - return indexMode.defaultSourceMode().name(); - }, "index.mapping.source.mode", new Setting.Validator<>() { - @Override - public void validate(Mode value) { - - } - - @Override - public Iterator> settings() { - List> res = List.of(IndexSettings.MODE); - return res.iterator(); - } - }, Setting.Property.Final, Setting.Property.IndexScope); - public static final String DEPRECATION_WARNING = "Configuring source mode in mappings is deprecated and will be removed " + "in future versions. Use [index.mapping.source.mode] index setting instead."; @@ -273,8 +255,8 @@ public SourceFieldMapper build() { private Mode resolveSourceMode() { // If the `index.mapping.source.mode` exists it takes precedence to determine the source mode for `_source` // otherwise the mode is determined according to `_source.mode`. - if (INDEX_MAPPER_SOURCE_MODE_SETTING.exists(settings)) { - return INDEX_MAPPER_SOURCE_MODE_SETTING.get(settings); + if (IndexSettings.INDEX_MAPPER_SOURCE_MODE_SETTING.exists(settings)) { + return IndexSettings.INDEX_MAPPER_SOURCE_MODE_SETTING.get(settings); } // If `_source.mode` is not set we need to apply a default according to index mode. @@ -306,7 +288,7 @@ private static SourceFieldMapper resolveStaticInstance(final Mode sourceMode) { return DEFAULT; } - final Mode settingSourceMode = INDEX_MAPPER_SOURCE_MODE_SETTING.get(c.getSettings()); + final Mode settingSourceMode = IndexSettings.INDEX_MAPPER_SOURCE_MODE_SETTING.get(c.getSettings()); // Needed for bwc so that "mode" is not serialized in case of standard index with stored source. if (indexMode == IndexMode.STANDARD && settingSourceMode == Mode.STORED) { return DEFAULT; @@ -491,11 +473,11 @@ public boolean isSynthetic() { } public static boolean isSynthetic(IndexSettings indexSettings) { - return INDEX_MAPPER_SOURCE_MODE_SETTING.get(indexSettings.getSettings()) == SourceFieldMapper.Mode.SYNTHETIC; + return IndexSettings.INDEX_MAPPER_SOURCE_MODE_SETTING.get(indexSettings.getSettings()) == SourceFieldMapper.Mode.SYNTHETIC; } public static boolean isStored(IndexSettings indexSettings) { - return INDEX_MAPPER_SOURCE_MODE_SETTING.get(indexSettings.getSettings()) == Mode.STORED; + return IndexSettings.INDEX_MAPPER_SOURCE_MODE_SETTING.get(indexSettings.getSettings()) == Mode.STORED; } public boolean isDisabled() { diff --git a/server/src/main/java/org/elasticsearch/snapshots/RestoreService.java b/server/src/main/java/org/elasticsearch/snapshots/RestoreService.java index 5aec8e0e3253c..1e3c85513d4d6 100644 --- a/server/src/main/java/org/elasticsearch/snapshots/RestoreService.java +++ b/server/src/main/java/org/elasticsearch/snapshots/RestoreService.java @@ -71,7 +71,6 @@ import org.elasticsearch.index.IndexVersion; import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.mapper.Mapping; -import org.elasticsearch.index.mapper.SourceFieldMapper; import org.elasticsearch.index.shard.IndexLongFieldRange; import org.elasticsearch.index.shard.IndexShard; import org.elasticsearch.index.shard.ShardId; @@ -159,7 +158,7 @@ public final class RestoreService implements ClusterStateApplier { SETTING_CREATION_DATE, SETTING_HISTORY_UUID, IndexSettings.MODE.getKey(), - SourceFieldMapper.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), + IndexSettings.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), IndexSettings.RECOVERY_USE_SYNTHETIC_SOURCE_SETTING.getKey(), IndexSortConfig.INDEX_SORT_FIELD_SETTING.getKey(), IndexSortConfig.INDEX_SORT_ORDER_SETTING.getKey(), diff --git a/server/src/test/java/org/elasticsearch/action/admin/cluster/stats/MappingStatsTests.java b/server/src/test/java/org/elasticsearch/action/admin/cluster/stats/MappingStatsTests.java index 96954458c18c4..0617053769da0 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/cluster/stats/MappingStatsTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/cluster/stats/MappingStatsTests.java @@ -17,8 +17,8 @@ import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.Writeable.Reader; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.IndexVersion; -import org.elasticsearch.index.mapper.SourceFieldMapper; import org.elasticsearch.script.Script; import org.elasticsearch.tasks.TaskCancelledException; import org.elasticsearch.test.AbstractWireSerializingTestCase; @@ -586,7 +586,7 @@ public void testSourceModes() { int numDisabledIndices = randomIntBetween(1, 5); for (int i = 0; i < numSyntheticIndices; i++) { IndexMetadata.Builder indexMetadata = new IndexMetadata.Builder("foo-synthetic-" + i).settings( - indexSettings(IndexVersion.current(), 4, 1).put(SourceFieldMapper.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), "synthetic") + indexSettings(IndexVersion.current(), 4, 1).put(IndexSettings.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), "synthetic") ); builder.put(indexMetadata); } @@ -594,7 +594,7 @@ public void testSourceModes() { IndexMetadata.Builder indexMetadata; if (randomBoolean()) { indexMetadata = new IndexMetadata.Builder("foo-stored-" + i).settings( - indexSettings(IndexVersion.current(), 4, 1).put(SourceFieldMapper.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), "stored") + indexSettings(IndexVersion.current(), 4, 1).put(IndexSettings.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), "stored") ); } else { indexMetadata = new IndexMetadata.Builder("foo-stored-" + i).settings(indexSettings(IndexVersion.current(), 4, 1)); @@ -603,7 +603,7 @@ public void testSourceModes() { } for (int i = 0; i < numDisabledIndices; i++) { IndexMetadata.Builder indexMetadata = new IndexMetadata.Builder("foo-disabled-" + i).settings( - indexSettings(IndexVersion.current(), 4, 1).put(SourceFieldMapper.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), "disabled") + indexSettings(IndexVersion.current(), 4, 1).put(IndexSettings.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), "disabled") ); builder.put(indexMetadata); } diff --git a/server/src/test/java/org/elasticsearch/index/engine/LuceneSyntheticSourceChangesSnapshotTests.java b/server/src/test/java/org/elasticsearch/index/engine/LuceneSyntheticSourceChangesSnapshotTests.java index 2a6c3428d6d45..a5d5d9b210e33 100644 --- a/server/src/test/java/org/elasticsearch/index/engine/LuceneSyntheticSourceChangesSnapshotTests.java +++ b/server/src/test/java/org/elasticsearch/index/engine/LuceneSyntheticSourceChangesSnapshotTests.java @@ -19,14 +19,12 @@ import java.io.IOException; -import static org.elasticsearch.index.mapper.SourceFieldMapper.INDEX_MAPPER_SOURCE_MODE_SETTING; - public class LuceneSyntheticSourceChangesSnapshotTests extends SearchBasedChangesSnapshotTests { @Override protected Settings indexSettings() { return Settings.builder() .put(super.indexSettings()) - .put(INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), SourceFieldMapper.Mode.SYNTHETIC.name()) + .put(IndexSettings.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), SourceFieldMapper.Mode.SYNTHETIC.name()) .put(IndexSettings.RECOVERY_USE_SYNTHETIC_SOURCE_SETTING.getKey(), true) .build(); } diff --git a/server/src/test/java/org/elasticsearch/index/engine/TranslogOperationAsserterTests.java b/server/src/test/java/org/elasticsearch/index/engine/TranslogOperationAsserterTests.java index b764bce464d15..d0455c14bd784 100644 --- a/server/src/test/java/org/elasticsearch/index/engine/TranslogOperationAsserterTests.java +++ b/server/src/test/java/org/elasticsearch/index/engine/TranslogOperationAsserterTests.java @@ -24,8 +24,6 @@ import java.io.IOException; -import static org.elasticsearch.index.mapper.SourceFieldMapper.INDEX_MAPPER_SOURCE_MODE_SETTING; - public class TranslogOperationAsserterTests extends EngineTestCase { @Override @@ -33,7 +31,7 @@ protected Settings indexSettings() { return Settings.builder() .put(super.indexSettings()) .put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), true) - .put(INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), SourceFieldMapper.Mode.SYNTHETIC.name()) + .put(IndexSettings.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), SourceFieldMapper.Mode.SYNTHETIC.name()) .put(IndexSettings.RECOVERY_USE_SYNTHETIC_SOURCE_SETTING.getKey(), true) .build(); } @@ -57,10 +55,10 @@ EngineConfig engineConfig(boolean useSyntheticSource) { EngineConfig config = engine.config(); Settings.Builder settings = Settings.builder().put(config.getIndexSettings().getSettings()); if (useSyntheticSource) { - settings.put(INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), SourceFieldMapper.Mode.SYNTHETIC.name()); + settings.put(IndexSettings.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), SourceFieldMapper.Mode.SYNTHETIC.name()); settings.put(IndexSettings.RECOVERY_USE_SYNTHETIC_SOURCE_SETTING.getKey(), true); } else { - settings.put(INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), SourceFieldMapper.Mode.STORED.name()); + settings.put(IndexSettings.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), SourceFieldMapper.Mode.STORED.name()); settings.put(IndexSettings.RECOVERY_USE_SYNTHETIC_SOURCE_SETTING.getKey(), false); } IndexMetadata imd = IndexMetadata.builder(config.getIndexSettings().getIndexMetadata()).settings(settings).build(); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/IgnoredSourceFieldMapperConfigurationTests.java b/server/src/test/java/org/elasticsearch/index/mapper/IgnoredSourceFieldMapperConfigurationTests.java index 8646e1b66dcb0..1ba5f423d4b03 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/IgnoredSourceFieldMapperConfigurationTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/IgnoredSourceFieldMapperConfigurationTests.java @@ -12,6 +12,7 @@ import org.apache.lucene.index.DirectoryReader; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.core.CheckedConsumer; +import org.elasticsearch.index.IndexSettings; import org.elasticsearch.xcontent.XContentBuilder; import java.io.IOException; @@ -130,7 +131,7 @@ private MapperService mapperServiceWithCustomSettings( for (var entry : customSettings.entrySet()) { settings.put(entry.getKey(), entry.getValue()); } - settings.put(SourceFieldMapper.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), SourceFieldMapper.Mode.SYNTHETIC); + settings.put(IndexSettings.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), SourceFieldMapper.Mode.SYNTHETIC); return createMapperService(settings.build(), mapping(mapping)); } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/SourceFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/SourceFieldMapperTests.java index b7693513a434d..bc560d94b8f52 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/SourceFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/SourceFieldMapperTests.java @@ -427,7 +427,7 @@ public void testRecoverySourceWitInvalidSettings() { { Settings settings = Settings.builder() - .put(SourceFieldMapper.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), SourceFieldMapper.Mode.STORED.toString()) + .put(IndexSettings.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), SourceFieldMapper.Mode.STORED.toString()) .put(IndexSettings.RECOVERY_USE_SYNTHETIC_SOURCE_SETTING.getKey(), true) .build(); IllegalArgumentException exc = expectThrows( @@ -470,7 +470,7 @@ public void testRecoverySourceWitInvalidSettings() { public void testRecoverySourceWithSyntheticSource() throws IOException { { Settings settings = Settings.builder() - .put(SourceFieldMapper.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), SourceFieldMapper.Mode.SYNTHETIC.toString()) + .put(IndexSettings.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), SourceFieldMapper.Mode.SYNTHETIC.toString()) .build(); MapperService mapperService = createMapperService(settings, topMapping(b -> {})); DocumentMapper docMapper = mapperService.documentMapper(); @@ -480,7 +480,7 @@ public void testRecoverySourceWithSyntheticSource() throws IOException { } { Settings settings = Settings.builder() - .put(SourceFieldMapper.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), SourceFieldMapper.Mode.SYNTHETIC.toString()) + .put(IndexSettings.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), SourceFieldMapper.Mode.SYNTHETIC.toString()) .put(IndexSettings.RECOVERY_USE_SYNTHETIC_SOURCE_SETTING.getKey(), true) .build(); MapperService mapperService = createMapperService(settings, topMapping(b -> {})); @@ -539,7 +539,7 @@ public void testStandardIndexModeWithSourceModeSetting() throws IOException { final XContentBuilder mappings = topMapping(b -> {}); final Settings settings = Settings.builder() .put(IndexSettings.MODE.getKey(), IndexMode.STANDARD.name()) - .put(SourceFieldMapper.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), SourceFieldMapper.Mode.SYNTHETIC) + .put(IndexSettings.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), SourceFieldMapper.Mode.SYNTHETIC) .build(); final MapperService mapperService = createMapperService(settings, mappings); DocumentMapper docMapper = mapperService.documentMapper(); @@ -549,7 +549,7 @@ public void testStandardIndexModeWithSourceModeSetting() throws IOException { final XContentBuilder mappings = topMapping(b -> {}); final Settings settings = Settings.builder() .put(IndexSettings.MODE.getKey(), IndexMode.STANDARD.name()) - .put(SourceFieldMapper.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), SourceFieldMapper.Mode.STORED) + .put(IndexSettings.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), SourceFieldMapper.Mode.STORED) .build(); final MapperService mapperService = createMapperService(settings, mappings); final DocumentMapper docMapper = mapperService.documentMapper(); @@ -559,7 +559,7 @@ public void testStandardIndexModeWithSourceModeSetting() throws IOException { final XContentBuilder mappings = topMapping(b -> {}); final Settings settings = Settings.builder() .put(IndexSettings.MODE.getKey(), IndexMode.STANDARD.name()) - .put(SourceFieldMapper.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), SourceFieldMapper.Mode.DISABLED) + .put(IndexSettings.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), SourceFieldMapper.Mode.DISABLED) .build(); final MapperService mapperService = createMapperService(settings, mappings); final DocumentMapper docMapper = mapperService.documentMapper(); @@ -571,7 +571,7 @@ public void testStandardIndexModeWithSourceModeSetting() throws IOException { final XContentBuilder mappings = topMapping(b -> {}); final Settings settings = Settings.builder() .put(IndexSettings.MODE.getKey(), IndexMode.LOGSDB.name()) - .put(SourceFieldMapper.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), SourceFieldMapper.Mode.SYNTHETIC) + .put(IndexSettings.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), SourceFieldMapper.Mode.SYNTHETIC) .build(); final MapperService mapperService = createMapperService(settings, mappings); DocumentMapper docMapper = mapperService.documentMapper(); @@ -581,7 +581,7 @@ public void testStandardIndexModeWithSourceModeSetting() throws IOException { final XContentBuilder mappings = topMapping(b -> {}); final Settings settings = Settings.builder() .put(IndexSettings.MODE.getKey(), IndexMode.LOGSDB.name()) - .put(SourceFieldMapper.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), SourceFieldMapper.Mode.STORED) + .put(IndexSettings.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), SourceFieldMapper.Mode.STORED) .build(); final MapperService mapperService = createMapperService(settings, mappings); final DocumentMapper docMapper = mapperService.documentMapper(); @@ -591,7 +591,7 @@ public void testStandardIndexModeWithSourceModeSetting() throws IOException { final XContentBuilder mappings = topMapping(b -> {}); final Settings settings = Settings.builder() .put(IndexSettings.MODE.getKey(), IndexMode.LOGSDB.name()) - .put(SourceFieldMapper.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), SourceFieldMapper.Mode.DISABLED) + .put(IndexSettings.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), SourceFieldMapper.Mode.DISABLED) .build(); var ex = expectThrows(MapperParsingException.class, () -> createMapperService(settings, mappings)); assertEquals("Failed to parse mapping: _source can not be disabled in index using [logsdb] index mode", ex.getMessage()); @@ -613,7 +613,7 @@ public void testStandardIndexModeWithSourceModeSetting() throws IOException { """; final Settings settings = Settings.builder() .put(IndexSettings.MODE.getKey(), IndexMode.TIME_SERIES.name()) - .put(SourceFieldMapper.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), SourceFieldMapper.Mode.SYNTHETIC) + .put(IndexSettings.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), SourceFieldMapper.Mode.SYNTHETIC) .put(IndexMetadata.INDEX_ROUTING_PATH.getKey(), "routing_field") .build(); final MapperService mapperService = createMapperService(settings, mappings); @@ -635,7 +635,7 @@ public void testStandardIndexModeWithSourceModeSetting() throws IOException { """; final Settings settings = Settings.builder() .put(IndexSettings.MODE.getKey(), IndexMode.TIME_SERIES.name()) - .put(SourceFieldMapper.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), SourceFieldMapper.Mode.STORED) + .put(IndexSettings.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), SourceFieldMapper.Mode.STORED) .put(IndexMetadata.INDEX_ROUTING_PATH.getKey(), "routing_field") .build(); final MapperService mapperService = createMapperService(settings, mappings); @@ -657,7 +657,7 @@ public void testStandardIndexModeWithSourceModeSetting() throws IOException { """; final Settings settings = Settings.builder() .put(IndexSettings.MODE.getKey(), IndexMode.TIME_SERIES.name()) - .put(SourceFieldMapper.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), SourceFieldMapper.Mode.DISABLED) + .put(IndexSettings.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), SourceFieldMapper.Mode.DISABLED) .put(IndexMetadata.INDEX_ROUTING_PATH.getKey(), "routing_field") .build(); var ex = expectThrows(MapperParsingException.class, () -> createMapperService(settings, mappings)); @@ -668,7 +668,7 @@ public void testStandardIndexModeWithSourceModeSetting() throws IOException { { final XContentBuilder mappings = topMapping(b -> {}); final Settings settings = Settings.builder() - .put(SourceFieldMapper.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), SourceFieldMapper.Mode.SYNTHETIC) + .put(IndexSettings.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), SourceFieldMapper.Mode.SYNTHETIC) .build(); final MapperService mapperService = createMapperService(settings, mappings); DocumentMapper docMapper = mapperService.documentMapper(); @@ -677,7 +677,7 @@ public void testStandardIndexModeWithSourceModeSetting() throws IOException { { final XContentBuilder mappings = topMapping(b -> {}); final Settings settings = Settings.builder() - .put(SourceFieldMapper.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), SourceFieldMapper.Mode.STORED) + .put(IndexSettings.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), SourceFieldMapper.Mode.STORED) .build(); final MapperService mapperService = createMapperService(settings, mappings); final DocumentMapper docMapper = mapperService.documentMapper(); @@ -686,7 +686,7 @@ public void testStandardIndexModeWithSourceModeSetting() throws IOException { { final XContentBuilder mappings = topMapping(b -> {}); final Settings settings = Settings.builder() - .put(SourceFieldMapper.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), SourceFieldMapper.Mode.DISABLED) + .put(IndexSettings.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), SourceFieldMapper.Mode.DISABLED) .build(); final MapperService mapperService = createMapperService(settings, mappings); final DocumentMapper docMapper = mapperService.documentMapper(); diff --git a/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperTestCase.java b/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperTestCase.java index bce88a2a8d2f7..b33abc2d781e8 100644 --- a/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperTestCase.java @@ -1180,7 +1180,7 @@ private void assertSyntheticSourceWithTranslogSnapshot(SyntheticSourceSupport su var firstExample = support.example(1); int maxDocs = randomIntBetween(20, 50); var settings = Settings.builder() - .put(SourceFieldMapper.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), SourceFieldMapper.Mode.SYNTHETIC) + .put(IndexSettings.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), SourceFieldMapper.Mode.SYNTHETIC) .put(IndexSettings.RECOVERY_USE_SYNTHETIC_SOURCE_SETTING.getKey(), true) .build(); var mapperService = createMapperService(getVersion(), settings, () -> true, mapping(b -> { diff --git a/x-pack/plugin/ccr/qa/multi-cluster/src/test/java/org/elasticsearch/xpack/ccr/FollowIndexIT.java b/x-pack/plugin/ccr/qa/multi-cluster/src/test/java/org/elasticsearch/xpack/ccr/FollowIndexIT.java index 0bb4afe51b85a..e8e19bad2a7ef 100644 --- a/x-pack/plugin/ccr/qa/multi-cluster/src/test/java/org/elasticsearch/xpack/ccr/FollowIndexIT.java +++ b/x-pack/plugin/ccr/qa/multi-cluster/src/test/java/org/elasticsearch/xpack/ccr/FollowIndexIT.java @@ -368,7 +368,7 @@ public void testSyntheticSource() throws Exception { if ("leader".equals(targetCluster)) { logger.info("Running against leader cluster"); Settings settings = Settings.builder() - .put(SourceFieldMapper.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), SourceFieldMapper.Mode.SYNTHETIC) + .put(IndexSettings.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), SourceFieldMapper.Mode.SYNTHETIC) .build(); createIndex(adminClient(), leaderIndexName, settings, """ "properties": {"kwd": {"type": "keyword"}}}""", null); diff --git a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/TransportResumeFollowActionTests.java b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/TransportResumeFollowActionTests.java index 357e1bca38e8f..6bc25215855ee 100644 --- a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/TransportResumeFollowActionTests.java +++ b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/TransportResumeFollowActionTests.java @@ -18,7 +18,6 @@ import org.elasticsearch.index.MapperTestUtils; import org.elasticsearch.index.mapper.IgnoredSourceFieldMapper; import org.elasticsearch.index.mapper.MapperService; -import org.elasticsearch.index.mapper.SourceFieldMapper; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.xpack.ccr.Ccr; import org.elasticsearch.xpack.ccr.CcrSettings; @@ -335,7 +334,7 @@ public void testDynamicIndexSettingsAreClassified() { replicatedSettings.add(IndexSettings.PREFER_ILM_SETTING); replicatedSettings.add(IgnoredSourceFieldMapper.SKIP_IGNORED_SOURCE_READ_SETTING); replicatedSettings.add(IgnoredSourceFieldMapper.SKIP_IGNORED_SOURCE_WRITE_SETTING); - replicatedSettings.add(SourceFieldMapper.INDEX_MAPPER_SOURCE_MODE_SETTING); + replicatedSettings.add(IndexSettings.INDEX_MAPPER_SOURCE_MODE_SETTING); for (Setting setting : IndexScopedSettings.BUILT_IN_INDEX_SETTINGS) { // removed settings have no effect, they are only there for BWC diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/snapshots/sourceonly/SourceOnlySnapshotShardTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/snapshots/sourceonly/SourceOnlySnapshotShardTests.java index 81d194ce84131..a2cdfff2e2c80 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/snapshots/sourceonly/SourceOnlySnapshotShardTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/snapshots/sourceonly/SourceOnlySnapshotShardTests.java @@ -46,6 +46,7 @@ import org.elasticsearch.core.IOUtils; import org.elasticsearch.env.Environment; import org.elasticsearch.env.TestEnvironment; +import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.IndexVersion; import org.elasticsearch.index.VersionType; import org.elasticsearch.index.engine.Engine; @@ -53,7 +54,6 @@ import org.elasticsearch.index.engine.InternalEngineFactory; import org.elasticsearch.index.fieldvisitor.FieldsVisitor; import org.elasticsearch.index.mapper.SeqNoFieldMapper; -import org.elasticsearch.index.mapper.SourceFieldMapper; import org.elasticsearch.index.mapper.SourceToParse; import org.elasticsearch.index.seqno.RetentionLeaseSyncer; import org.elasticsearch.index.seqno.SeqNoStats; @@ -162,7 +162,7 @@ public void testSourceIncompleteSyntheticSourceNoDoc() throws IOException { .put(IndexMetadata.SETTING_VERSION_CREATED, IndexVersion.current()) .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) - .put(SourceFieldMapper.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), "synthetic") + .put(IndexSettings.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), "synthetic") .build(); IndexMetadata metadata = IndexMetadata.builder(shardRouting.getIndexName()).settings(settings).primaryTerm(0, primaryTerm).build(); IndexShard shard = newShard(shardRouting, metadata, null, new InternalEngineFactory()); diff --git a/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/LogsDBUsageTransportAction.java b/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/LogsDBUsageTransportAction.java index f4fa2a29d79a0..cdf02273b9df5 100644 --- a/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/LogsDBUsageTransportAction.java +++ b/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/LogsDBUsageTransportAction.java @@ -16,6 +16,7 @@ import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.features.FeatureService; import org.elasticsearch.index.IndexMode; +import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.mapper.SourceFieldMapper; import org.elasticsearch.injection.guice.Inject; import org.elasticsearch.monitor.metrics.IndexModeStatsActionType; @@ -29,8 +30,6 @@ import org.elasticsearch.xpack.core.action.XPackUsageFeatureTransportAction; import org.elasticsearch.xpack.core.application.LogsDBFeatureSetUsage; -import static org.elasticsearch.index.mapper.SourceFieldMapper.INDEX_MAPPER_SOURCE_MODE_SETTING; - public class LogsDBUsageTransportAction extends XPackUsageFeatureTransportAction { private final ClusterService clusterService; private final FeatureService featureService; @@ -71,7 +70,7 @@ protected void masterOperation( for (IndexMetadata indexMetadata : state.metadata()) { if (indexMetadata.getIndexMode() == IndexMode.LOGSDB) { numIndices++; - if (INDEX_MAPPER_SOURCE_MODE_SETTING.get(indexMetadata.getSettings()) == SourceFieldMapper.Mode.SYNTHETIC) { + if (IndexSettings.INDEX_MAPPER_SOURCE_MODE_SETTING.get(indexMetadata.getSettings()) == SourceFieldMapper.Mode.SYNTHETIC) { numIndicesWithSyntheticSources++; } } diff --git a/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/LogsdbIndexModeSettingsProvider.java b/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/LogsdbIndexModeSettingsProvider.java index f95b64f8c0ec9..df1fb31bf9e0e 100644 --- a/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/LogsdbIndexModeSettingsProvider.java +++ b/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/LogsdbIndexModeSettingsProvider.java @@ -115,7 +115,7 @@ && matchesLogsPattern(dataStreamName)) { if (settingsBuilder == null) { settingsBuilder = Settings.builder(); } - settingsBuilder.put(SourceFieldMapper.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), SourceFieldMapper.Mode.STORED.toString()); + settingsBuilder.put(IndexSettings.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), SourceFieldMapper.Mode.STORED.toString()); } } @@ -187,7 +187,7 @@ boolean newIndexHasSyntheticSourceUsage( try { var tmpIndexMetadata = buildIndexMetadataForMapperService(indexName, templateIndexMode, indexTemplateAndCreateRequestSettings); var indexMode = tmpIndexMetadata.getIndexMode(); - if (SourceFieldMapper.INDEX_MAPPER_SOURCE_MODE_SETTING.exists(tmpIndexMetadata.getSettings()) + if (IndexSettings.INDEX_MAPPER_SOURCE_MODE_SETTING.exists(tmpIndexMetadata.getSettings()) || indexMode == IndexMode.LOGSDB || indexMode == IndexMode.TIME_SERIES) { // In case when index mode is tsdb or logsdb and only _source.mode mapping attribute is specified, then the default @@ -195,7 +195,7 @@ boolean newIndexHasSyntheticSourceUsage( // then configuring the index.mapping.source.mode setting to stored has no effect. Additionally _source.mode can't be set // to disabled, because that isn't allowed with logsdb/tsdb. In other words setting index.mapping.source.mode setting to // stored when _source.mode mapping attribute is stored is fine as it has no effect, but avoids creating MapperService. - var sourceMode = SourceFieldMapper.INDEX_MAPPER_SOURCE_MODE_SETTING.get(tmpIndexMetadata.getSettings()); + var sourceMode = IndexSettings.INDEX_MAPPER_SOURCE_MODE_SETTING.get(tmpIndexMetadata.getSettings()); return sourceMode == SourceFieldMapper.Mode.SYNTHETIC; } diff --git a/x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/LegacyLicenceIntegrationTests.java b/x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/LegacyLicenceIntegrationTests.java index f8f307b572f33..b2a533f6b76c9 100644 --- a/x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/LegacyLicenceIntegrationTests.java +++ b/x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/LegacyLicenceIntegrationTests.java @@ -9,7 +9,7 @@ import org.elasticsearch.action.admin.indices.settings.get.GetSettingsRequest; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.index.mapper.SourceFieldMapper; +import org.elasticsearch.index.IndexSettings; import org.elasticsearch.license.AbstractLicensesIntegrationTestCase; import org.elasticsearch.license.GetFeatureUsageRequest; import org.elasticsearch.license.GetFeatureUsageResponse; @@ -104,11 +104,11 @@ public void testSyntheticSourceUsageTracksBothLegacyAndRegularFeature() throws E } private void createIndexWithSyntheticSourceAndAssertExpectedType(String indexName, String expectedType) { - var settings = Settings.builder().put(SourceFieldMapper.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), "synthetic").build(); + var settings = Settings.builder().put(IndexSettings.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), "synthetic").build(); createIndex(indexName, settings); var response = admin().indices().getSettings(new GetSettingsRequest().indices(indexName)).actionGet(); assertThat( - response.getIndexToSettings().get(indexName).get(SourceFieldMapper.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey()), + response.getIndexToSettings().get(indexName).get(IndexSettings.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey()), equalTo(expectedType) ); } diff --git a/x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/LogsdbIndexModeSettingsProviderTests.java b/x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/LogsdbIndexModeSettingsProviderTests.java index 5d3cb7b2a9967..e7e8acd9b65d8 100644 --- a/x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/LogsdbIndexModeSettingsProviderTests.java +++ b/x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/LogsdbIndexModeSettingsProviderTests.java @@ -615,7 +615,7 @@ public void testGetAdditionalIndexSettingsDowngradeFromSyntheticSource() throws ); Metadata metadata = mb.build(); LogsdbIndexModeSettingsProvider provider = withSyntheticSourceDemotionSupport(false); - Settings settings = builder().put(SourceFieldMapper.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), SourceFieldMapper.Mode.SYNTHETIC) + Settings settings = builder().put(IndexSettings.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), SourceFieldMapper.Mode.SYNTHETIC) .build(); Settings result = provider.getAdditionalIndexSettings( @@ -641,7 +641,7 @@ public void testGetAdditionalIndexSettingsDowngradeFromSyntheticSource() throws List.of() ); assertThat(result.size(), equalTo(1)); - assertEquals(SourceFieldMapper.Mode.STORED, SourceFieldMapper.INDEX_MAPPER_SOURCE_MODE_SETTING.get(result)); + assertEquals(SourceFieldMapper.Mode.STORED, IndexSettings.INDEX_MAPPER_SOURCE_MODE_SETTING.get(result)); assertThat(newMapperServiceCounter.get(), equalTo(0)); result = provider.getAdditionalIndexSettings( @@ -654,7 +654,7 @@ public void testGetAdditionalIndexSettingsDowngradeFromSyntheticSource() throws List.of() ); assertThat(result.size(), equalTo(1)); - assertEquals(SourceFieldMapper.Mode.STORED, SourceFieldMapper.INDEX_MAPPER_SOURCE_MODE_SETTING.get(result)); + assertEquals(SourceFieldMapper.Mode.STORED, IndexSettings.INDEX_MAPPER_SOURCE_MODE_SETTING.get(result)); assertThat(newMapperServiceCounter.get(), equalTo(0)); result = provider.getAdditionalIndexSettings( @@ -667,7 +667,7 @@ public void testGetAdditionalIndexSettingsDowngradeFromSyntheticSource() throws List.of() ); assertThat(result.size(), equalTo(1)); - assertEquals(SourceFieldMapper.Mode.STORED, SourceFieldMapper.INDEX_MAPPER_SOURCE_MODE_SETTING.get(result)); + assertEquals(SourceFieldMapper.Mode.STORED, IndexSettings.INDEX_MAPPER_SOURCE_MODE_SETTING.get(result)); assertThat(newMapperServiceCounter.get(), equalTo(0)); } @@ -720,7 +720,7 @@ public void testGetAdditionalIndexSettingsDowngradeFromSyntheticSourceFileMatch( List.of() ); assertThat(result.size(), equalTo(2)); - assertEquals(SourceFieldMapper.Mode.STORED, SourceFieldMapper.INDEX_MAPPER_SOURCE_MODE_SETTING.get(result)); + assertEquals(SourceFieldMapper.Mode.STORED, IndexSettings.INDEX_MAPPER_SOURCE_MODE_SETTING.get(result)); assertEquals(IndexMode.LOGSDB, IndexSettings.MODE.get(result)); result = provider.getAdditionalIndexSettings( diff --git a/x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/LogsdbIndexSettingsProviderLegacyLicenseTests.java b/x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/LogsdbIndexSettingsProviderLegacyLicenseTests.java index 8a4adf18b3e67..0c45782b2a7fb 100644 --- a/x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/LogsdbIndexSettingsProviderLegacyLicenseTests.java +++ b/x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/LogsdbIndexSettingsProviderLegacyLicenseTests.java @@ -10,9 +10,9 @@ import org.elasticsearch.cluster.metadata.DataStream; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.IndexMode; +import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.IndexVersion; import org.elasticsearch.index.MapperTestUtils; -import org.elasticsearch.index.mapper.SourceFieldMapper; import org.elasticsearch.license.License; import org.elasticsearch.license.LicenseService; import org.elasticsearch.license.XPackLicenseState; @@ -57,16 +57,16 @@ public void setup() throws Exception { } public void testGetAdditionalIndexSettingsDefault() { - Settings settings = Settings.builder().put(SourceFieldMapper.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), "SYNTHETIC").build(); + Settings settings = Settings.builder().put(IndexSettings.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), "SYNTHETIC").build(); String dataStreamName = "metrics-my-app"; String indexName = DataStream.getDefaultBackingIndexName(dataStreamName, 0); var result = provider.getAdditionalIndexSettings(indexName, dataStreamName, null, null, null, settings, List.of()); assertThat(result.size(), equalTo(1)); - assertThat(result.get(SourceFieldMapper.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey()), equalTo("STORED")); + assertThat(result.get(IndexSettings.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey()), equalTo("STORED")); } public void testGetAdditionalIndexSettingsApm() throws IOException { - Settings settings = Settings.builder().put(SourceFieldMapper.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), "SYNTHETIC").build(); + Settings settings = Settings.builder().put(IndexSettings.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), "SYNTHETIC").build(); String dataStreamName = "metrics-apm.app.test"; String indexName = DataStream.getDefaultBackingIndexName(dataStreamName, 0); var result = provider.getAdditionalIndexSettings(indexName, dataStreamName, null, null, null, settings, List.of()); @@ -74,7 +74,7 @@ public void testGetAdditionalIndexSettingsApm() throws IOException { } public void testGetAdditionalIndexSettingsProfiling() throws IOException { - Settings settings = Settings.builder().put(SourceFieldMapper.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), "SYNTHETIC").build(); + Settings settings = Settings.builder().put(IndexSettings.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), "SYNTHETIC").build(); for (String dataStreamName : new String[] { "profiling-metrics", "profiling-events" }) { String indexName = DataStream.getDefaultBackingIndexName(dataStreamName, 0); var result = provider.getAdditionalIndexSettings(indexName, dataStreamName, null, null, null, settings, List.of()); @@ -88,7 +88,7 @@ public void testGetAdditionalIndexSettingsProfiling() throws IOException { } public void testGetAdditionalIndexSettingsTsdb() throws IOException { - Settings settings = Settings.builder().put(SourceFieldMapper.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), "SYNTHETIC").build(); + Settings settings = Settings.builder().put(IndexSettings.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), "SYNTHETIC").build(); String dataStreamName = "metrics-my-app"; String indexName = DataStream.getDefaultBackingIndexName(dataStreamName, 0); var result = provider.getAdditionalIndexSettings(indexName, dataStreamName, IndexMode.TIME_SERIES, null, null, settings, List.of()); @@ -116,11 +116,11 @@ public void testGetAdditionalIndexSettingsTsdbAfterCutoffDate() throws Exception IndexVersion::current ); - Settings settings = Settings.builder().put(SourceFieldMapper.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), "SYNTHETIC").build(); + Settings settings = Settings.builder().put(IndexSettings.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), "SYNTHETIC").build(); String dataStreamName = "metrics-my-app"; String indexName = DataStream.getDefaultBackingIndexName(dataStreamName, 0); var result = provider.getAdditionalIndexSettings(indexName, dataStreamName, IndexMode.TIME_SERIES, null, null, settings, List.of()); assertThat(result.size(), equalTo(1)); - assertThat(result.get(SourceFieldMapper.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey()), equalTo("STORED")); + assertThat(result.get(IndexSettings.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey()), equalTo("STORED")); } } From 7bcd75157c153e3258f35a2cc065495ad0f7e2d1 Mon Sep 17 00:00:00 2001 From: Salvatore Campagna Date: Wed, 8 Jan 2025 15:23:58 +0100 Subject: [PATCH 09/54] fix: preserve histogram array in synthetic source --- .../xpack/analytics/mapper/HistogramFieldMapperTests.java | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/mapper/HistogramFieldMapperTests.java b/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/mapper/HistogramFieldMapperTests.java index fd2888129817f..c9bccee3414e5 100644 --- a/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/mapper/HistogramFieldMapperTests.java +++ b/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/mapper/HistogramFieldMapperTests.java @@ -407,10 +407,12 @@ public void testArrayValueSyntheticSource() throws Exception { @Override protected SyntheticSourceSupport syntheticSourceSupport(boolean ignoreMalformed) { - return new HistogramFieldSyntheticSourceSupport(ignoreMalformed); + return new HistogramFieldSyntheticSourceSupport(ignoreMalformed, "all"); } - private record HistogramFieldSyntheticSourceSupport(boolean ignoreMalformed) implements SyntheticSourceSupport { + private record HistogramFieldSyntheticSourceSupport(boolean ignoreMalformed, String syntheticSourceKeep) + implements + SyntheticSourceSupport { @Override public SyntheticSourceExample example(int maxVals) { if (randomBoolean()) { @@ -442,6 +444,7 @@ private void mapping(XContentBuilder b) throws IOException { if (ignoreMalformed) { b.field("ignore_malformed", true); } + b.field("synthetic_source_keep", syntheticSourceKeep); } @Override From 0cfda3642aad1e2616f496a9c99078ca798d8cc2 Mon Sep 17 00:00:00 2001 From: Salvatore Campagna Date: Wed, 8 Jan 2025 15:37:24 +0100 Subject: [PATCH 10/54] fix: missing _recovery_source field when using synthetic source --- .../test/update/100_synthetic_source.yml | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/update/100_synthetic_source.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/update/100_synthetic_source.yml index f4894692b6cad..219bc52c4e28c 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/update/100_synthetic_source.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/update/100_synthetic_source.yml @@ -6,8 +6,8 @@ setup: --- keyword: - requires: - cluster_features: ["gte_v8.4.0"] - reason: introduced in 8.4.0 + cluster_features: [ "mapper.synthetic_recovery_source" ] + reason: requires synthetic recovery source - do: indices.create: @@ -60,13 +60,14 @@ keyword: index: test run_expensive_tasks: true - is_false: test.fields._source - - is_true: test.fields._recovery_source + # When synthetic source is used there is no _recovery_source field + - match: { test.fields._recovery_source: null } --- stored text: - requires: - cluster_features: ["gte_v8.5.0"] - reason: introduced in 8.5.0 + cluster_features: [ "mapper.synthetic_recovery_source" ] + reason: requires synthetic recovery source - do: indices.create: @@ -121,4 +122,5 @@ stored text: index: test run_expensive_tasks: true - is_false: test.fields._source - - is_true: test.fields._recovery_source + # When synthetic source is used there is no _recovery_source field + - match: { test.fields._recovery_source: null } From 95e43e440a9b4474e2e69c57be7f4e09c6e2d017 Mon Sep 17 00:00:00 2001 From: Salvatore Campagna Date: Wed, 8 Jan 2025 16:06:47 +0100 Subject: [PATCH 11/54] Revert "fix: preserve histogram array in synthetic source" This reverts commit 7bcd75157c153e3258f35a2cc065495ad0f7e2d1. --- .../xpack/analytics/mapper/HistogramFieldMapperTests.java | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/mapper/HistogramFieldMapperTests.java b/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/mapper/HistogramFieldMapperTests.java index c9bccee3414e5..fd2888129817f 100644 --- a/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/mapper/HistogramFieldMapperTests.java +++ b/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/mapper/HistogramFieldMapperTests.java @@ -407,12 +407,10 @@ public void testArrayValueSyntheticSource() throws Exception { @Override protected SyntheticSourceSupport syntheticSourceSupport(boolean ignoreMalformed) { - return new HistogramFieldSyntheticSourceSupport(ignoreMalformed, "all"); + return new HistogramFieldSyntheticSourceSupport(ignoreMalformed); } - private record HistogramFieldSyntheticSourceSupport(boolean ignoreMalformed, String syntheticSourceKeep) - implements - SyntheticSourceSupport { + private record HistogramFieldSyntheticSourceSupport(boolean ignoreMalformed) implements SyntheticSourceSupport { @Override public SyntheticSourceExample example(int maxVals) { if (randomBoolean()) { @@ -444,7 +442,6 @@ private void mapping(XContentBuilder b) throws IOException { if (ignoreMalformed) { b.field("ignore_malformed", true); } - b.field("synthetic_source_keep", syntheticSourceKeep); } @Override From 8dd466b352c07584e6ab222fff8ee3ce5cf67044 Mon Sep 17 00:00:00 2001 From: Salvatore Campagna Date: Wed, 8 Jan 2025 17:19:44 +0100 Subject: [PATCH 12/54] fix: disable testing for empty list --- .../xpack/analytics/mapper/HistogramFieldMapperTests.java | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/mapper/HistogramFieldMapperTests.java b/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/mapper/HistogramFieldMapperTests.java index fd2888129817f..1b5779d6193aa 100644 --- a/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/mapper/HistogramFieldMapperTests.java +++ b/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/mapper/HistogramFieldMapperTests.java @@ -454,4 +454,9 @@ public List invalidExample() throws IOException { public void testSyntheticSourceKeepArrays() { // The mapper expects to parse an array of values by default, it's not compatible with array of arrays. } + + @Override + protected boolean supportsEmptyInputArray() { + return false; + } } From b2bbbf7451a26f60a3920be1b7b126edadabfce0 Mon Sep 17 00:00:00 2001 From: Salvatore Campagna Date: Thu, 9 Jan 2025 15:14:25 +0100 Subject: [PATCH 13/54] fix: exclude _recovery_source_size --- .../elasticsearch/index/mapper/MapperServiceTestCase.java | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperServiceTestCase.java b/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperServiceTestCase.java index b9356bc4b5633..6352c2d179234 100644 --- a/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperServiceTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperServiceTestCase.java @@ -892,8 +892,11 @@ protected void validateRoundTripReader(String syntheticSource, DirectoryReader r throws IOException { assertReaderEquals( "round trip " + syntheticSource, - new FieldMaskingReader(SourceFieldMapper.RECOVERY_SOURCE_NAME, reader), - new FieldMaskingReader(SourceFieldMapper.RECOVERY_SOURCE_NAME, roundTripReader) + new FieldMaskingReader(Set.of(SourceFieldMapper.RECOVERY_SOURCE_NAME, SourceFieldMapper.RECOVERY_SOURCE_SIZE_NAME), reader), + new FieldMaskingReader( + Set.of(SourceFieldMapper.RECOVERY_SOURCE_NAME, SourceFieldMapper.RECOVERY_SOURCE_SIZE_NAME), + roundTripReader + ) ); } From 1b6b524c1091201a1b460dc935c28f5b1e4081c2 Mon Sep 17 00:00:00 2001 From: Salvatore Campagna Date: Thu, 9 Jan 2025 15:20:17 +0100 Subject: [PATCH 14/54] fix: exclude _recovery_source_size --- .../index/mapper/IgnoredSourceFieldMapperTests.java | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/server/src/test/java/org/elasticsearch/index/mapper/IgnoredSourceFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/IgnoredSourceFieldMapperTests.java index 14902aa419b9f..ee9e657e919fc 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/IgnoredSourceFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/IgnoredSourceFieldMapperTests.java @@ -2410,8 +2410,14 @@ protected void validateRoundTripReader(String syntheticSource, DirectoryReader r // and since the copy is exact, contents of ignored source are different. assertReaderEquals( "round trip " + syntheticSource, - new FieldMaskingReader(Set.of(SourceFieldMapper.RECOVERY_SOURCE_NAME, IgnoredSourceFieldMapper.NAME), reader), - new FieldMaskingReader(Set.of(SourceFieldMapper.RECOVERY_SOURCE_NAME, IgnoredSourceFieldMapper.NAME), roundTripReader) + new FieldMaskingReader( + Set.of(SourceFieldMapper.RECOVERY_SOURCE_NAME, IgnoredSourceFieldMapper.NAME, SourceFieldMapper.RECOVERY_SOURCE_SIZE_NAME), + reader + ), + new FieldMaskingReader( + Set.of(SourceFieldMapper.RECOVERY_SOURCE_NAME, IgnoredSourceFieldMapper.NAME, SourceFieldMapper.RECOVERY_SOURCE_SIZE_NAME), + roundTripReader + ) ); } } From b87504d2448109e78a75caf645d3608d2425d2e9 Mon Sep 17 00:00:00 2001 From: Salvatore Campagna Date: Thu, 9 Jan 2025 15:47:48 +0100 Subject: [PATCH 15/54] fix: disable rest compatibility test after dropping _recovery_source field --- rest-api-spec/build.gradle | 2 ++ 1 file changed, 2 insertions(+) diff --git a/rest-api-spec/build.gradle b/rest-api-spec/build.gradle index a104ec675adc2..e077f51464955 100644 --- a/rest-api-spec/build.gradle +++ b/rest-api-spec/build.gradle @@ -79,4 +79,6 @@ tasks.named("yamlRestCompatTestTransform").configure ({ task -> task.skipTest("search.vectors/110_knn_query_with_filter/PRE_FILTER: pre-filter across multiple aliases", "waiting for #118774 backport") task.skipTest("search.vectors/160_knn_query_missing_params/kNN search in a dis_max query - missing num_candidates", "waiting for #118774 backport") task.skipTest("search.highlight/30_max_analyzed_offset/Plain highlighter with max_analyzed_offset < 0 should FAIL", "semantics of test has changed") + task.skipTest("update/100_synthetic_source/stored test", "After introducing #114618 _recovery_source field does not exist anymore when using synthetic source") + task.skipTest("update/100_synthetic_source/keyword", "After introducing #114618 _recovery_source field does not exist anymore when using synthetic source") }) From 278ab193382fea5b12dcb3008ea091dc61bf928a Mon Sep 17 00:00:00 2001 From: Salvatore Campagna Date: Thu, 9 Jan 2025 15:59:50 +0100 Subject: [PATCH 16/54] fix: no need to disable empty list test --- .../xpack/analytics/mapper/HistogramFieldMapperTests.java | 5 ----- 1 file changed, 5 deletions(-) diff --git a/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/mapper/HistogramFieldMapperTests.java b/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/mapper/HistogramFieldMapperTests.java index 1b5779d6193aa..fd2888129817f 100644 --- a/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/mapper/HistogramFieldMapperTests.java +++ b/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/mapper/HistogramFieldMapperTests.java @@ -454,9 +454,4 @@ public List invalidExample() throws IOException { public void testSyntheticSourceKeepArrays() { // The mapper expects to parse an array of values by default, it's not compatible with array of arrays. } - - @Override - protected boolean supportsEmptyInputArray() { - return false; - } } From 7118892231dcccfcf337ea17d3364885b644f76e Mon Sep 17 00:00:00 2001 From: Salvatore Campagna Date: Thu, 9 Jan 2025 23:02:44 +0100 Subject: [PATCH 17/54] test: synthetic source recovery and source mode --- .../elasticsearch/index/IndexSettings.java | 3 + .../test/60_synthetic_source_recovery.yml | 261 ++++++++++++++++++ 2 files changed, 264 insertions(+) create mode 100644 x-pack/plugin/logsdb/src/yamlRestTest/resources/rest-api-spec/test/60_synthetic_source_recovery.yml diff --git a/server/src/main/java/org/elasticsearch/index/IndexSettings.java b/server/src/main/java/org/elasticsearch/index/IndexSettings.java index 6b13f86fabe8e..cbbb4ead28061 100644 --- a/server/src/main/java/org/elasticsearch/index/IndexSettings.java +++ b/server/src/main/java/org/elasticsearch/index/IndexSettings.java @@ -709,6 +709,9 @@ public Iterator> settings() { public static final Setting RECOVERY_USE_SYNTHETIC_SOURCE_SETTING = Setting.boolSetting( "index.recovery.use_synthetic_source", settings -> { + if (IndexMetadata.SETTING_INDEX_VERSION_CREATED.get(settings).before(IndexVersions.USE_SYNTHETIC_SOURCE_FOR_RECOVERY)) { + return String.valueOf(true); + } final SourceFieldMapper.Mode sourceMode = INDEX_MAPPER_SOURCE_MODE_SETTING.get(settings); return String.valueOf(sourceMode == SourceFieldMapper.Mode.SYNTHETIC); }, diff --git a/x-pack/plugin/logsdb/src/yamlRestTest/resources/rest-api-spec/test/60_synthetic_source_recovery.yml b/x-pack/plugin/logsdb/src/yamlRestTest/resources/rest-api-spec/test/60_synthetic_source_recovery.yml new file mode 100644 index 0000000000000..cc2216997c6d3 --- /dev/null +++ b/x-pack/plugin/logsdb/src/yamlRestTest/resources/rest-api-spec/test/60_synthetic_source_recovery.yml @@ -0,0 +1,261 @@ +--- +synthetic recovery for synthetic source mode index: + - requires: + cluster_features: [ "mapper.synthetic_recovery_source" ] + reason: requires synthetic recovery source + + - do: + indices.create: + index: test_synthetic_recovery + body: + settings: + index: + mapping.source.mode: synthetic + + - do: + indices.get_settings: + index: test_synthetic_recovery + include_defaults: true + + - match: { test_synthetic_recovery.settings.index.mapping.source.mode: synthetic } + - match: { test_synthetic_recovery.defaults.index.recovery.use_synthetic_source: "true" } + +--- +synthetic recovery for stored source mode index: + - requires: + cluster_features: [ "mapper.synthetic_recovery_source" ] + reason: requires synthetic recovery source + + - do: + indices.create: + index: test_stored_recovery + body: + settings: + index: + mapping.source.mode: stored + + - do: + indices.get_settings: + index: test_stored_recovery + include_defaults: true + + - match: { test_stored_recovery.settings.index.mapping.source.mode: stored } + - match: { test_stored_recovery.defaults.index.recovery.use_synthetic_source: "false" } + +--- +synthetic recovery for disabled source mode index: + - requires: + cluster_features: [ "mapper.synthetic_recovery_source" ] + reason: requires synthetic recovery source + + - do: + indices.create: + index: test_disabled_recovery + body: + settings: + index: + mapping.source.mode: disabled + + - do: + indices.get_settings: + index: test_disabled_recovery + include_defaults: true + + - match: { test_disabled_recovery.settings.index.mapping.source.mode: disabled } + - match: { test_disabled_recovery.defaults.index.recovery.use_synthetic_source: "false" } + +--- +synthetic recovery for standard index: + - requires: + cluster_features: [ "mapper.synthetic_recovery_source" ] + reason: requires synthetic recovery source + + - do: + indices.create: + index: test_standard_index_recovery + body: + settings: + index: + mode: standard + + - do: + indices.get_settings: + index: test_standard_index_recovery + include_defaults: true + + - match: { test_standard_index_recovery.defaults.index.recovery.use_synthetic_source: "false" } + +--- +synthetic recovery for logsdb index: + - requires: + cluster_features: [ "mapper.synthetic_recovery_source" ] + reason: requires synthetic recovery source + + - do: + indices.create: + index: test_logsdb_index_recovery + body: + settings: + index: + mode: logsdb + + - do: + indices.get_settings: + index: test_logsdb_index_recovery + include_defaults: true + + - match: { test_logsdb_index_recovery.defaults.index.recovery.use_synthetic_source: "true" } + +--- +synthetic recovery for time_series index: + - requires: + cluster_features: [ "mapper.synthetic_recovery_source" ] + reason: requires synthetic recovery source + + - do: + indices.create: + index: test_time_series_index_recovery + body: + settings: + index: + mode: time_series + routing_path: [ keyword ] + time_series: + start_time: 2021-04-28T00:00:00Z + end_time: 2021-04-29T00:00:00Z + mappings: + properties: + keyword: + type: keyword + time_series_dimension: true + + - do: + indices.get_settings: + index: test_time_series_index_recovery + include_defaults: true + + - match: { test_time_series_index_recovery.defaults.index.recovery.use_synthetic_source: "true" } + +--- +override synthetic recovery for synthetic source mode index: + - requires: + cluster_features: [ "mapper.synthetic_recovery_source" ] + reason: requires synthetic recovery source + + - do: + indices.create: + index: test_synthetic_recovery_override + body: + settings: + index: + mapping.source.mode: synthetic + recovery.use_synthetic_source: false + + - do: + indices.get_settings: + index: test_synthetic_recovery_override + include_defaults: true + + - match: { test_synthetic_recovery_override.settings.index.mapping.source.mode: synthetic } + - match: { test_synthetic_recovery_override.settings.index.recovery.use_synthetic_source: "false" } + +--- +override synthetic recovery for stored source mode index: + - requires: + cluster_features: [ "mapper.synthetic_recovery_source" ] + reason: requires synthetic recovery source + + - do: + catch: bad_request + indices.create: + index: test_stored_recovery_override + body: + settings: + index: + mapping.source.mode: stored + recovery.use_synthetic_source: true + +--- +override synthetic recovery for disabled source mode index: + - requires: + cluster_features: [ "mapper.synthetic_recovery_source" ] + reason: requires synthetic recovery source + + - do: + catch: bad_request + indices.create: + index: test_disabled_recovery_override + body: + settings: + index: + mapping.source.mode: disabled + recovery.use_synthetic_source: true + +--- +override synthetic recovery for standard index: + - requires: + cluster_features: [ "mapper.synthetic_recovery_source" ] + reason: requires synthetic recovery source + + - do: + catch: bad_request + indices.create: + index: test_standard_index_recovery_override + body: + settings: + index: + mode: standard + recovery.use_synthetic_source: true + +--- +override synthetic recovery for logsdb index: + - requires: + cluster_features: [ "mapper.synthetic_recovery_source" ] + reason: requires synthetic recovery source + + - do: + indices.create: + index: test_logsdb_index_recovery_override + body: + settings: + index: + mode: logsdb + recovery.use_synthetic_source: false + + - do: + indices.get_settings: + index: test_logsdb_index_recovery_override + include_defaults: true + + - match: { test_logsdb_index_recovery_override.settings.index.recovery.use_synthetic_source: "false" } + +--- +override synthetic recovery for time_series index: + - requires: + cluster_features: [ "mapper.synthetic_recovery_source" ] + reason: requires synthetic recovery source + + - do: + indices.create: + index: test_time_series_index_recovery_override + body: + settings: + index: + mode: time_series + recovery.use_synthetic_source: false + routing_path: [ keyword ] + time_series: + start_time: 2021-04-28T00:00:00Z + end_time: 2021-04-29T00:00:00Z + mappings: + properties: + keyword: + type: keyword + time_series_dimension: true + + - do: + indices.get_settings: + index: test_time_series_index_recovery_override + include_defaults: true + + - match: { test_time_series_index_recovery_override.settings.index.recovery.use_synthetic_source: "false" } From d36a6baf1df9d48675a23b2b420c4e4fa5af7971 Mon Sep 17 00:00:00 2001 From: Salvatore Campagna Date: Fri, 10 Jan 2025 09:33:25 +0100 Subject: [PATCH 18/54] fix: false for bwc --- server/src/main/java/org/elasticsearch/index/IndexSettings.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/main/java/org/elasticsearch/index/IndexSettings.java b/server/src/main/java/org/elasticsearch/index/IndexSettings.java index cbbb4ead28061..51ec4cff6c201 100644 --- a/server/src/main/java/org/elasticsearch/index/IndexSettings.java +++ b/server/src/main/java/org/elasticsearch/index/IndexSettings.java @@ -710,7 +710,7 @@ public Iterator> settings() { "index.recovery.use_synthetic_source", settings -> { if (IndexMetadata.SETTING_INDEX_VERSION_CREATED.get(settings).before(IndexVersions.USE_SYNTHETIC_SOURCE_FOR_RECOVERY)) { - return String.valueOf(true); + return String.valueOf(false); } final SourceFieldMapper.Mode sourceMode = INDEX_MAPPER_SOURCE_MODE_SETTING.get(settings); return String.valueOf(sourceMode == SourceFieldMapper.Mode.SYNTHETIC); From 857883259bcc6afbfcd89d6f2fd44c4132e525d8 Mon Sep 17 00:00:00 2001 From: Salvatore Campagna Date: Fri, 10 Jan 2025 10:02:18 +0100 Subject: [PATCH 19/54] fix: typo test vs text --- rest-api-spec/build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rest-api-spec/build.gradle b/rest-api-spec/build.gradle index e077f51464955..c972747fe02de 100644 --- a/rest-api-spec/build.gradle +++ b/rest-api-spec/build.gradle @@ -79,6 +79,6 @@ tasks.named("yamlRestCompatTestTransform").configure ({ task -> task.skipTest("search.vectors/110_knn_query_with_filter/PRE_FILTER: pre-filter across multiple aliases", "waiting for #118774 backport") task.skipTest("search.vectors/160_knn_query_missing_params/kNN search in a dis_max query - missing num_candidates", "waiting for #118774 backport") task.skipTest("search.highlight/30_max_analyzed_offset/Plain highlighter with max_analyzed_offset < 0 should FAIL", "semantics of test has changed") - task.skipTest("update/100_synthetic_source/stored test", "After introducing #114618 _recovery_source field does not exist anymore when using synthetic source") + task.skipTest("update/100_synthetic_source/stored text", "After introducing #114618 _recovery_source field does not exist anymore when using synthetic source") task.skipTest("update/100_synthetic_source/keyword", "After introducing #114618 _recovery_source field does not exist anymore when using synthetic source") }) From 2a509740aabd7818d636568b443356e214dbec9f Mon Sep 17 00:00:00 2001 From: Salvatore Campagna Date: Fri, 10 Jan 2025 10:11:23 +0100 Subject: [PATCH 20/54] fix: setting is final --- .../src/main/java/org/elasticsearch/index/IndexSettings.java | 3 --- 1 file changed, 3 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/index/IndexSettings.java b/server/src/main/java/org/elasticsearch/index/IndexSettings.java index a0a9054fa0180..cc70aac63f8f6 100644 --- a/server/src/main/java/org/elasticsearch/index/IndexSettings.java +++ b/server/src/main/java/org/elasticsearch/index/IndexSettings.java @@ -709,9 +709,6 @@ public Iterator> settings() { public static final Setting RECOVERY_USE_SYNTHETIC_SOURCE_SETTING = Setting.boolSetting( "index.recovery.use_synthetic_source", settings -> { - if (IndexMetadata.SETTING_INDEX_VERSION_CREATED.get(settings).before(IndexVersions.USE_SYNTHETIC_SOURCE_FOR_RECOVERY)) { - return String.valueOf(false); - } final SourceFieldMapper.Mode sourceMode = INDEX_MAPPER_SOURCE_MODE_SETTING.get(settings); return String.valueOf(sourceMode == SourceFieldMapper.Mode.SYNTHETIC); }, From 063531d5c5ac0b4a047351775c3013ba4ee15a33 Mon Sep 17 00:00:00 2001 From: Salvatore Campagna Date: Fri, 10 Jan 2025 10:56:59 +0100 Subject: [PATCH 21/54] fix: synthetic recovery index version --- .../java/org/elasticsearch/index/IndexSettings.java | 10 ++++++++-- .../java/org/elasticsearch/index/IndexVersions.java | 2 ++ 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/index/IndexSettings.java b/server/src/main/java/org/elasticsearch/index/IndexSettings.java index cc70aac63f8f6..6eb9f2a1e5766 100644 --- a/server/src/main/java/org/elasticsearch/index/IndexSettings.java +++ b/server/src/main/java/org/elasticsearch/index/IndexSettings.java @@ -709,8 +709,14 @@ public Iterator> settings() { public static final Setting RECOVERY_USE_SYNTHETIC_SOURCE_SETTING = Setting.boolSetting( "index.recovery.use_synthetic_source", settings -> { - final SourceFieldMapper.Mode sourceMode = INDEX_MAPPER_SOURCE_MODE_SETTING.get(settings); - return String.valueOf(sourceMode == SourceFieldMapper.Mode.SYNTHETIC); + if (IndexMetadata.SETTING_INDEX_VERSION_CREATED.get(settings) + .onOrAfter(IndexVersions.USE_SYNTHETIC_SOURCE_FOR_RECOVERY_BY_DEFAULT) + || SETTING_INDEX_VERSION_CREATED.get(settings) + .between(IndexVersions.USE_SYNTHETIC_SOURCE_FOR_RECOVERY_BY_DEFAULT_BACKPORT, IndexVersions.UPGRADE_TO_LUCENE_10_0_0)) { + final SourceFieldMapper.Mode sourceMode = INDEX_MAPPER_SOURCE_MODE_SETTING.get(settings); + return String.valueOf(sourceMode == SourceFieldMapper.Mode.SYNTHETIC); + } + return String.valueOf(false); }, new Setting.Validator<>() { @Override diff --git a/server/src/main/java/org/elasticsearch/index/IndexVersions.java b/server/src/main/java/org/elasticsearch/index/IndexVersions.java index 8d6404e0530e5..d03f14d7ef47e 100644 --- a/server/src/main/java/org/elasticsearch/index/IndexVersions.java +++ b/server/src/main/java/org/elasticsearch/index/IndexVersions.java @@ -134,12 +134,14 @@ private static Version parseUnchecked(String version) { public static final IndexVersion USE_SYNTHETIC_SOURCE_FOR_RECOVERY_BACKPORT = def(8_522_00_0, Version.LUCENE_9_12_0); public static final IndexVersion UPGRADE_TO_LUCENE_9_12_1 = def(8_523_00_0, parseUnchecked("9.12.1")); public static final IndexVersion INFERENCE_METADATA_FIELDS_BACKPORT = def(8_524_00_0, parseUnchecked("9.12.1")); + public static final IndexVersion USE_SYNTHETIC_SOURCE_FOR_RECOVERY_BY_DEFAULT_BACKPORT = def(8_525_00_0, parseUnchecked("9.12.1")); public static final IndexVersion UPGRADE_TO_LUCENE_10_0_0 = def(9_000_00_0, Version.LUCENE_10_0_0); public static final IndexVersion LOGSDB_DEFAULT_IGNORE_DYNAMIC_BEYOND_LIMIT = def(9_001_00_0, Version.LUCENE_10_0_0); public static final IndexVersion TIME_BASED_K_ORDERED_DOC_ID = def(9_002_00_0, Version.LUCENE_10_0_0); public static final IndexVersion DEPRECATE_SOURCE_MODE_MAPPER = def(9_003_00_0, Version.LUCENE_10_0_0); public static final IndexVersion USE_SYNTHETIC_SOURCE_FOR_RECOVERY = def(9_004_00_0, Version.LUCENE_10_0_0); public static final IndexVersion INFERENCE_METADATA_FIELDS = def(9_005_00_0, Version.LUCENE_10_0_0); + public static final IndexVersion USE_SYNTHETIC_SOURCE_FOR_RECOVERY_BY_DEFAULT = def(9_006_00_0, Version.LUCENE_10_0_0); /* * STOP! READ THIS FIRST! No, really, From 8a6418a5bfc45a35e78ba8d41ddaaee81b56cbf2 Mon Sep 17 00:00:00 2001 From: Salvatore Campagna Date: Fri, 10 Jan 2025 12:51:14 +0100 Subject: [PATCH 22/54] fix: recover source field unvailable with synthetic source --- .../index/mapper/SourceFieldMapperTests.java | 21 +++++-------------- 1 file changed, 5 insertions(+), 16 deletions(-) diff --git a/server/src/test/java/org/elasticsearch/index/mapper/SourceFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/SourceFieldMapperTests.java index bc560d94b8f52..496b4b9cae02f 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/SourceFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/SourceFieldMapperTests.java @@ -475,8 +475,7 @@ public void testRecoverySourceWithSyntheticSource() throws IOException { MapperService mapperService = createMapperService(settings, topMapping(b -> {})); DocumentMapper docMapper = mapperService.documentMapper(); ParsedDocument doc = docMapper.parse(source(b -> b.field("field1", "value1"))); - assertNotNull(doc.rootDoc().getField("_recovery_source")); - assertThat(doc.rootDoc().getField("_recovery_source").binaryValue(), equalTo(new BytesRef("{\"field1\":\"value1\"}"))); + assertNull(doc.rootDoc().getField("_recovery_source")); } { Settings settings = Settings.builder() @@ -507,8 +506,7 @@ public void testRecoverySourceWithLogs() throws IOException { MapperService mapperService = createMapperService(settings, mapping(b -> {})); DocumentMapper docMapper = mapperService.documentMapper(); ParsedDocument doc = docMapper.parse(source(b -> { b.field("@timestamp", "2012-02-13"); })); - assertNotNull(doc.rootDoc().getField("_recovery_source")); - assertThat(doc.rootDoc().getField("_recovery_source").binaryValue(), equalTo(new BytesRef("{\"@timestamp\":\"2012-02-13\"}"))); + assertNull(doc.rootDoc().getField("_recovery_source")); } { Settings settings = Settings.builder() @@ -701,8 +699,7 @@ public void testRecoverySourceWithLogsCustom() throws IOException { MapperService mapperService = createMapperService(settings, mappings); DocumentMapper docMapper = mapperService.documentMapper(); ParsedDocument doc = docMapper.parse(source(b -> { b.field("@timestamp", "2012-02-13"); })); - assertNotNull(doc.rootDoc().getField("_recovery_source")); - assertThat(doc.rootDoc().getField("_recovery_source").binaryValue(), equalTo(new BytesRef("{\"@timestamp\":\"2012-02-13\"}"))); + assertNull(doc.rootDoc().getField("_recovery_source")); } { Settings settings = Settings.builder() @@ -728,11 +725,7 @@ public void testRecoverySourceWithTimeSeries() throws IOException { })); DocumentMapper docMapper = mapperService.documentMapper(); ParsedDocument doc = docMapper.parse(source("123", b -> b.field("@timestamp", "2012-02-13").field("field", "value1"), null)); - assertNotNull(doc.rootDoc().getField("_recovery_source")); - assertThat( - doc.rootDoc().getField("_recovery_source").binaryValue(), - equalTo(new BytesRef("{\"@timestamp\":\"2012-02-13\",\"field\":\"value1\"}")) - ); + assertNull(doc.rootDoc().getField("_recovery_source")); } { Settings settings = Settings.builder() @@ -776,11 +769,7 @@ public void testRecoverySourceWithTimeSeriesCustom() throws IOException { MapperService mapperService = createMapperService(settings, mappings); DocumentMapper docMapper = mapperService.documentMapper(); ParsedDocument doc = docMapper.parse(source("123", b -> b.field("@timestamp", "2012-02-13").field("field", "value1"), null)); - assertNotNull(doc.rootDoc().getField("_recovery_source")); - assertThat( - doc.rootDoc().getField("_recovery_source").binaryValue(), - equalTo(new BytesRef("{\"@timestamp\":\"2012-02-13\",\"field\":\"value1\"}")) - ); + assertNull(doc.rootDoc().getField("_recovery_source")); } { Settings settings = Settings.builder() From b05e0612c29afa982870019227afc2db4f1ee5e4 Mon Sep 17 00:00:00 2001 From: Salvatore Campagna Date: Fri, 10 Jan 2025 16:31:01 +0100 Subject: [PATCH 23/54] fix: synthetic recovery only for stateful --- .../src/main/java/org/elasticsearch/index/IndexSettings.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/index/IndexSettings.java b/server/src/main/java/org/elasticsearch/index/IndexSettings.java index 680ced239e8d7..830a722711153 100644 --- a/server/src/main/java/org/elasticsearch/index/IndexSettings.java +++ b/server/src/main/java/org/elasticsearch/index/IndexSettings.java @@ -725,8 +725,9 @@ public Iterator> settings() { public static final Setting RECOVERY_USE_SYNTHETIC_SOURCE_SETTING = Setting.boolSetting( "index.recovery.use_synthetic_source", settings -> { - if (IndexMetadata.SETTING_INDEX_VERSION_CREATED.get(settings) - .onOrAfter(IndexVersions.USE_SYNTHETIC_SOURCE_FOR_RECOVERY_BY_DEFAULT) + if (DiscoveryNode.isStateless(settings) == false + && IndexMetadata.SETTING_INDEX_VERSION_CREATED.get(settings) + .onOrAfter(IndexVersions.USE_SYNTHETIC_SOURCE_FOR_RECOVERY_BY_DEFAULT) || SETTING_INDEX_VERSION_CREATED.get(settings) .between(IndexVersions.USE_SYNTHETIC_SOURCE_FOR_RECOVERY_BY_DEFAULT_BACKPORT, IndexVersions.UPGRADE_TO_LUCENE_10_0_0)) { final SourceFieldMapper.Mode sourceMode = INDEX_MAPPER_SOURCE_MODE_SETTING.get(settings); From 1b03ddb49813b3ce37df36bd32434d8103b0d9a5 Mon Sep 17 00:00:00 2001 From: Salvatore Campagna Date: Fri, 10 Jan 2025 18:58:12 +0100 Subject: [PATCH 24/54] fix: no recovery source for stateless --- .../main/java/org/elasticsearch/index/IndexSettings.java | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/index/IndexSettings.java b/server/src/main/java/org/elasticsearch/index/IndexSettings.java index 830a722711153..aa9b05d379aac 100644 --- a/server/src/main/java/org/elasticsearch/index/IndexSettings.java +++ b/server/src/main/java/org/elasticsearch/index/IndexSettings.java @@ -725,9 +725,11 @@ public Iterator> settings() { public static final Setting RECOVERY_USE_SYNTHETIC_SOURCE_SETTING = Setting.boolSetting( "index.recovery.use_synthetic_source", settings -> { - if (DiscoveryNode.isStateless(settings) == false - && IndexMetadata.SETTING_INDEX_VERSION_CREATED.get(settings) - .onOrAfter(IndexVersions.USE_SYNTHETIC_SOURCE_FOR_RECOVERY_BY_DEFAULT) + if (DiscoveryNode.isStateless(settings)) { + return String.valueOf(false); + } + if (IndexMetadata.SETTING_INDEX_VERSION_CREATED.get(settings) + .onOrAfter(IndexVersions.USE_SYNTHETIC_SOURCE_FOR_RECOVERY_BY_DEFAULT) || SETTING_INDEX_VERSION_CREATED.get(settings) .between(IndexVersions.USE_SYNTHETIC_SOURCE_FOR_RECOVERY_BY_DEFAULT_BACKPORT, IndexVersions.UPGRADE_TO_LUCENE_10_0_0)) { final SourceFieldMapper.Mode sourceMode = INDEX_MAPPER_SOURCE_MODE_SETTING.get(settings); From f3a892d63ac3619664c94dd487439a783afcf3b7 Mon Sep 17 00:00:00 2001 From: Salvatore Campagna Date: Mon, 13 Jan 2025 17:44:50 +0100 Subject: [PATCH 25/54] fix: remove stateless check --- .../src/main/java/org/elasticsearch/index/IndexSettings.java | 3 --- 1 file changed, 3 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/index/IndexSettings.java b/server/src/main/java/org/elasticsearch/index/IndexSettings.java index aa9b05d379aac..9d90529a137cb 100644 --- a/server/src/main/java/org/elasticsearch/index/IndexSettings.java +++ b/server/src/main/java/org/elasticsearch/index/IndexSettings.java @@ -1099,9 +1099,6 @@ public IndexSettings(final IndexMetadata indexMetadata, final Settings nodeSetti recoverySourceEnabled = RecoverySettings.INDICES_RECOVERY_SOURCE_ENABLED_SETTING.get(nodeSettings); recoverySourceSyntheticEnabled = scopedSettings.get(RECOVERY_USE_SYNTHETIC_SOURCE_SETTING); if (recoverySourceSyntheticEnabled) { - if (DiscoveryNode.isStateless(settings)) { - throw new IllegalArgumentException("synthetic recovery source is only allowed in stateful"); - } // Verify that all nodes can handle this setting if (version.before(IndexVersions.USE_SYNTHETIC_SOURCE_FOR_RECOVERY) && version.between( From 52df8c2796353aae599191a29a16621d36f3219a Mon Sep 17 00:00:00 2001 From: Salvatore Campagna Date: Tue, 14 Jan 2025 09:41:02 +0100 Subject: [PATCH 26/54] Revert "fix: remove stateless check" This reverts commit f3a892d63ac3619664c94dd487439a783afcf3b7. --- .../src/main/java/org/elasticsearch/index/IndexSettings.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/server/src/main/java/org/elasticsearch/index/IndexSettings.java b/server/src/main/java/org/elasticsearch/index/IndexSettings.java index 9d90529a137cb..aa9b05d379aac 100644 --- a/server/src/main/java/org/elasticsearch/index/IndexSettings.java +++ b/server/src/main/java/org/elasticsearch/index/IndexSettings.java @@ -1099,6 +1099,9 @@ public IndexSettings(final IndexMetadata indexMetadata, final Settings nodeSetti recoverySourceEnabled = RecoverySettings.INDICES_RECOVERY_SOURCE_ENABLED_SETTING.get(nodeSettings); recoverySourceSyntheticEnabled = scopedSettings.get(RECOVERY_USE_SYNTHETIC_SOURCE_SETTING); if (recoverySourceSyntheticEnabled) { + if (DiscoveryNode.isStateless(settings)) { + throw new IllegalArgumentException("synthetic recovery source is only allowed in stateful"); + } // Verify that all nodes can handle this setting if (version.before(IndexVersions.USE_SYNTHETIC_SOURCE_FOR_RECOVERY) && version.between( From 3b38eb4cc3ce603fb13d1f8e116fcb19c7054549 Mon Sep 17 00:00:00 2001 From: Salvatore Campagna Date: Tue, 14 Jan 2025 10:15:25 +0100 Subject: [PATCH 27/54] fix: remove serverless handling --- .../src/main/java/org/elasticsearch/index/IndexSettings.java | 3 --- 1 file changed, 3 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/index/IndexSettings.java b/server/src/main/java/org/elasticsearch/index/IndexSettings.java index aa9b05d379aac..680ced239e8d7 100644 --- a/server/src/main/java/org/elasticsearch/index/IndexSettings.java +++ b/server/src/main/java/org/elasticsearch/index/IndexSettings.java @@ -725,9 +725,6 @@ public Iterator> settings() { public static final Setting RECOVERY_USE_SYNTHETIC_SOURCE_SETTING = Setting.boolSetting( "index.recovery.use_synthetic_source", settings -> { - if (DiscoveryNode.isStateless(settings)) { - return String.valueOf(false); - } if (IndexMetadata.SETTING_INDEX_VERSION_CREATED.get(settings) .onOrAfter(IndexVersions.USE_SYNTHETIC_SOURCE_FOR_RECOVERY_BY_DEFAULT) || SETTING_INDEX_VERSION_CREATED.get(settings) From b8ac5ae260bdbdfc4af8a0fd3409db52a688a3db Mon Sep 17 00:00:00 2001 From: Salvatore Campagna Date: Tue, 14 Jan 2025 10:42:20 +0100 Subject: [PATCH 28/54] fix: synthetic recovery source false in serverless --- .../src/main/java/org/elasticsearch/index/IndexSettings.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/server/src/main/java/org/elasticsearch/index/IndexSettings.java b/server/src/main/java/org/elasticsearch/index/IndexSettings.java index 680ced239e8d7..59c78827b0908 100644 --- a/server/src/main/java/org/elasticsearch/index/IndexSettings.java +++ b/server/src/main/java/org/elasticsearch/index/IndexSettings.java @@ -1094,7 +1094,8 @@ public IndexSettings(final IndexMetadata indexMetadata, final Settings nodeSetti skipIgnoredSourceRead = scopedSettings.get(IgnoredSourceFieldMapper.SKIP_IGNORED_SOURCE_READ_SETTING); indexMappingSourceMode = scopedSettings.get(INDEX_MAPPER_SOURCE_MODE_SETTING); recoverySourceEnabled = RecoverySettings.INDICES_RECOVERY_SOURCE_ENABLED_SETTING.get(nodeSettings); - recoverySourceSyntheticEnabled = scopedSettings.get(RECOVERY_USE_SYNTHETIC_SOURCE_SETTING); + recoverySourceSyntheticEnabled = DiscoveryNode.isStateless(nodeSettings) == false + && scopedSettings.get(RECOVERY_USE_SYNTHETIC_SOURCE_SETTING); if (recoverySourceSyntheticEnabled) { if (DiscoveryNode.isStateless(settings)) { throw new IllegalArgumentException("synthetic recovery source is only allowed in stateful"); From fb302cf16c0842e6c8cdc0aad1e3c69371d21a95 Mon Sep 17 00:00:00 2001 From: elasticsearchmachine <58790826+elasticsearchmachine@users.noreply.github.com> Date: Wed, 15 Jan 2025 00:45:54 +1100 Subject: [PATCH 29/54] Mute org.elasticsearch.repositories.s3.S3RepositoryThirdPartyTests org.elasticsearch.repositories.s3.S3RepositoryThirdPartyTests #120115 --- muted-tests.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/muted-tests.yml b/muted-tests.yml index 0eb80b79b74c1..96f215b8b6815 100644 --- a/muted-tests.yml +++ b/muted-tests.yml @@ -269,6 +269,8 @@ tests: issue: https://github.com/elastic/elasticsearch/issues/120088 - class: org.elasticsearch.xpack.searchablesnapshots.minio.MinioSearchableSnapshotsIT issue: https://github.com/elastic/elasticsearch/issues/120101 +- class: org.elasticsearch.repositories.s3.S3RepositoryThirdPartyTests + issue: https://github.com/elastic/elasticsearch/issues/120115 # Examples: # From d3c31d94b0ba846ec2cc9d9c7886152eb6532aca Mon Sep 17 00:00:00 2001 From: elasticsearchmachine <58790826+elasticsearchmachine@users.noreply.github.com> Date: Wed, 15 Jan 2025 00:49:53 +1100 Subject: [PATCH 30/54] Mute org.elasticsearch.repositories.s3.RepositoryS3MinioBasicCredentialsRestIT org.elasticsearch.repositories.s3.RepositoryS3MinioBasicCredentialsRestIT #120117 --- muted-tests.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/muted-tests.yml b/muted-tests.yml index 96f215b8b6815..9cf2ef065f82f 100644 --- a/muted-tests.yml +++ b/muted-tests.yml @@ -271,6 +271,8 @@ tests: issue: https://github.com/elastic/elasticsearch/issues/120101 - class: org.elasticsearch.repositories.s3.S3RepositoryThirdPartyTests issue: https://github.com/elastic/elasticsearch/issues/120115 +- class: org.elasticsearch.repositories.s3.RepositoryS3MinioBasicCredentialsRestIT + issue: https://github.com/elastic/elasticsearch/issues/120117 # Examples: # From ee5ae64a25aba45ef53c2a4f05ad8a5ed16d81d0 Mon Sep 17 00:00:00 2001 From: Salvatore Campagna Date: Thu, 30 Jan 2025 12:10:33 +0100 Subject: [PATCH 31/54] fix: introduce a feature flag to gate usage of synthetic recovery --- .../elasticsearch/index/IndexSettings.java | 25 +++++++++++++------ 1 file changed, 17 insertions(+), 8 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/index/IndexSettings.java b/server/src/main/java/org/elasticsearch/index/IndexSettings.java index 59c78827b0908..b14a3e6fb8b0e 100644 --- a/server/src/main/java/org/elasticsearch/index/IndexSettings.java +++ b/server/src/main/java/org/elasticsearch/index/IndexSettings.java @@ -24,6 +24,7 @@ import org.elasticsearch.common.time.DateUtils; import org.elasticsearch.common.unit.ByteSizeUnit; import org.elasticsearch.common.unit.ByteSizeValue; +import org.elasticsearch.common.util.FeatureFlag; import org.elasticsearch.core.TimeValue; import org.elasticsearch.features.NodeFeature; import org.elasticsearch.index.mapper.IgnoredSourceFieldMapper; @@ -40,6 +41,7 @@ import java.util.List; import java.util.Locale; import java.util.Map; +import java.util.Objects; import java.util.concurrent.TimeUnit; import java.util.function.Consumer; @@ -722,17 +724,24 @@ public Iterator> settings() { Setting.Property.IndexScope ); + public static final FeatureFlag RECOVERY_USE_SYNTHETIC_SOURCE = new FeatureFlag("index_recovery_use_synthetic_source"); public static final Setting RECOVERY_USE_SYNTHETIC_SOURCE_SETTING = Setting.boolSetting( "index.recovery.use_synthetic_source", settings -> { - if (IndexMetadata.SETTING_INDEX_VERSION_CREATED.get(settings) - .onOrAfter(IndexVersions.USE_SYNTHETIC_SOURCE_FOR_RECOVERY_BY_DEFAULT) - || SETTING_INDEX_VERSION_CREATED.get(settings) - .between(IndexVersions.USE_SYNTHETIC_SOURCE_FOR_RECOVERY_BY_DEFAULT_BACKPORT, IndexVersions.UPGRADE_TO_LUCENE_10_0_0)) { - final SourceFieldMapper.Mode sourceMode = INDEX_MAPPER_SOURCE_MODE_SETTING.get(settings); - return String.valueOf(sourceMode == SourceFieldMapper.Mode.SYNTHETIC); - } - return String.valueOf(false); + boolean isSyntheticSourceRecoveryFeatureFlagEnabled = RECOVERY_USE_SYNTHETIC_SOURCE.isEnabled(); + boolean isNewIndexVersion = SETTING_INDEX_VERSION_CREATED.get(settings) + .onOrAfter(IndexVersions.USE_SYNTHETIC_SOURCE_FOR_RECOVERY_BY_DEFAULT); + boolean isIndexVersionInBackportRange = SETTING_INDEX_VERSION_CREATED.get(settings) + .between(IndexVersions.USE_SYNTHETIC_SOURCE_FOR_RECOVERY_BY_DEFAULT_BACKPORT, IndexVersions.UPGRADE_TO_LUCENE_10_0_0); + + boolean useSyntheticRecoverySource = isSyntheticSourceRecoveryFeatureFlagEnabled + && (isNewIndexVersion || isIndexVersionInBackportRange); + + return String.valueOf( + useSyntheticRecoverySource + && Objects.equals(INDEX_MAPPER_SOURCE_MODE_SETTING.get(settings), SourceFieldMapper.Mode.SYNTHETIC) + ); + }, new Setting.Validator<>() { @Override From 2cf0cedef1d0b49f10d7611a96dc81ed266dc80c Mon Sep 17 00:00:00 2001 From: Salvatore Campagna Date: Thu, 30 Jan 2025 18:04:51 +0100 Subject: [PATCH 32/54] Revert "fix: recover source field unvailable with synthetic source" This reverts commit 8a6418a5bfc45a35e78ba8d41ddaaee81b56cbf2. --- .../index/mapper/SourceFieldMapperTests.java | 21 ++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/server/src/test/java/org/elasticsearch/index/mapper/SourceFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/SourceFieldMapperTests.java index 142a883c4e694..8ad37908b2e9c 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/SourceFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/SourceFieldMapperTests.java @@ -489,7 +489,8 @@ public void testRecoverySourceWithSyntheticSource() throws IOException { MapperService mapperService = createMapperService(settings, topMapping(b -> {})); DocumentMapper docMapper = mapperService.documentMapper(); ParsedDocument doc = docMapper.parse(source(b -> b.field("field1", "value1"))); - assertNull(doc.rootDoc().getField("_recovery_source")); + assertNotNull(doc.rootDoc().getField("_recovery_source")); + assertThat(doc.rootDoc().getField("_recovery_source").binaryValue(), equalTo(new BytesRef("{\"field1\":\"value1\"}"))); } { Settings settings = Settings.builder() @@ -520,7 +521,8 @@ public void testRecoverySourceWithLogs() throws IOException { MapperService mapperService = createMapperService(settings, mapping(b -> {})); DocumentMapper docMapper = mapperService.documentMapper(); ParsedDocument doc = docMapper.parse(source(b -> { b.field("@timestamp", "2012-02-13"); })); - assertNull(doc.rootDoc().getField("_recovery_source")); + assertNotNull(doc.rootDoc().getField("_recovery_source")); + assertThat(doc.rootDoc().getField("_recovery_source").binaryValue(), equalTo(new BytesRef("{\"@timestamp\":\"2012-02-13\"}"))); } { Settings settings = Settings.builder() @@ -713,7 +715,8 @@ public void testRecoverySourceWithLogsCustom() throws IOException { MapperService mapperService = createMapperService(settings, mappings); DocumentMapper docMapper = mapperService.documentMapper(); ParsedDocument doc = docMapper.parse(source(b -> { b.field("@timestamp", "2012-02-13"); })); - assertNull(doc.rootDoc().getField("_recovery_source")); + assertNotNull(doc.rootDoc().getField("_recovery_source")); + assertThat(doc.rootDoc().getField("_recovery_source").binaryValue(), equalTo(new BytesRef("{\"@timestamp\":\"2012-02-13\"}"))); } { Settings settings = Settings.builder() @@ -739,7 +742,11 @@ public void testRecoverySourceWithTimeSeries() throws IOException { })); DocumentMapper docMapper = mapperService.documentMapper(); ParsedDocument doc = docMapper.parse(source("123", b -> b.field("@timestamp", "2012-02-13").field("field", "value1"), null)); - assertNull(doc.rootDoc().getField("_recovery_source")); + assertNotNull(doc.rootDoc().getField("_recovery_source")); + assertThat( + doc.rootDoc().getField("_recovery_source").binaryValue(), + equalTo(new BytesRef("{\"@timestamp\":\"2012-02-13\",\"field\":\"value1\"}")) + ); } { Settings settings = Settings.builder() @@ -783,7 +790,11 @@ public void testRecoverySourceWithTimeSeriesCustom() throws IOException { MapperService mapperService = createMapperService(settings, mappings); DocumentMapper docMapper = mapperService.documentMapper(); ParsedDocument doc = docMapper.parse(source("123", b -> b.field("@timestamp", "2012-02-13").field("field", "value1"), null)); - assertNull(doc.rootDoc().getField("_recovery_source")); + assertNotNull(doc.rootDoc().getField("_recovery_source")); + assertThat( + doc.rootDoc().getField("_recovery_source").binaryValue(), + equalTo(new BytesRef("{\"@timestamp\":\"2012-02-13\",\"field\":\"value1\"}")) + ); } { Settings settings = Settings.builder() From 62fb995f5c05f557bbcff15c2d6b55364a517fed Mon Sep 17 00:00:00 2001 From: Salvatore Campagna Date: Thu, 30 Jan 2025 18:17:33 +0100 Subject: [PATCH 33/54] fix: check test result based on synthetic source recovery feature flag --- .../index/mapper/SourceFieldMapperTests.java | 63 ++++++++++++++----- 1 file changed, 47 insertions(+), 16 deletions(-) diff --git a/server/src/test/java/org/elasticsearch/index/mapper/SourceFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/SourceFieldMapperTests.java index 8ad37908b2e9c..70010084cdb96 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/SourceFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/SourceFieldMapperTests.java @@ -489,8 +489,13 @@ public void testRecoverySourceWithSyntheticSource() throws IOException { MapperService mapperService = createMapperService(settings, topMapping(b -> {})); DocumentMapper docMapper = mapperService.documentMapper(); ParsedDocument doc = docMapper.parse(source(b -> b.field("field1", "value1"))); - assertNotNull(doc.rootDoc().getField("_recovery_source")); - assertThat(doc.rootDoc().getField("_recovery_source").binaryValue(), equalTo(new BytesRef("{\"field1\":\"value1\"}"))); + if (IndexSettings.RECOVERY_USE_SYNTHETIC_SOURCE.isEnabled() == false) { + // TODO: remove this if branch when removing the 'index_recovery_use_synthetic_source' feature flag + assertNotNull(doc.rootDoc().getField("_recovery_source")); + assertThat(doc.rootDoc().getField("_recovery_source").binaryValue(), equalTo(new BytesRef("{\"field1\":\"value1\"}"))); + } else { + assertNull(doc.rootDoc().getField("_recovery_source")); + } } { Settings settings = Settings.builder() @@ -521,8 +526,16 @@ public void testRecoverySourceWithLogs() throws IOException { MapperService mapperService = createMapperService(settings, mapping(b -> {})); DocumentMapper docMapper = mapperService.documentMapper(); ParsedDocument doc = docMapper.parse(source(b -> { b.field("@timestamp", "2012-02-13"); })); - assertNotNull(doc.rootDoc().getField("_recovery_source")); - assertThat(doc.rootDoc().getField("_recovery_source").binaryValue(), equalTo(new BytesRef("{\"@timestamp\":\"2012-02-13\"}"))); + if (IndexSettings.RECOVERY_USE_SYNTHETIC_SOURCE.isEnabled() == false) { + // TODO: remove this if branch when removing the 'index_recovery_use_synthetic_source' feature flag + assertNotNull(doc.rootDoc().getField("_recovery_source")); + assertThat( + doc.rootDoc().getField("_recovery_source").binaryValue(), + equalTo(new BytesRef("{\"@timestamp\":\"2012-02-13\"}")) + ); + } else { + assertNull(doc.rootDoc().getField("_recovery_source")); + } } { Settings settings = Settings.builder() @@ -715,8 +728,16 @@ public void testRecoverySourceWithLogsCustom() throws IOException { MapperService mapperService = createMapperService(settings, mappings); DocumentMapper docMapper = mapperService.documentMapper(); ParsedDocument doc = docMapper.parse(source(b -> { b.field("@timestamp", "2012-02-13"); })); - assertNotNull(doc.rootDoc().getField("_recovery_source")); - assertThat(doc.rootDoc().getField("_recovery_source").binaryValue(), equalTo(new BytesRef("{\"@timestamp\":\"2012-02-13\"}"))); + if (IndexSettings.RECOVERY_USE_SYNTHETIC_SOURCE.isEnabled() == false) { + // TODO: remove this if branch when removing the 'index_recovery_use_synthetic_source' feature flag + assertNotNull(doc.rootDoc().getField("_recovery_source")); + assertThat( + doc.rootDoc().getField("_recovery_source").binaryValue(), + equalTo(new BytesRef("{\"@timestamp\":\"2012-02-13\"}")) + ); + } else { + assertNull(doc.rootDoc().getField("_recovery_source")); + } } { Settings settings = Settings.builder() @@ -742,11 +763,16 @@ public void testRecoverySourceWithTimeSeries() throws IOException { })); DocumentMapper docMapper = mapperService.documentMapper(); ParsedDocument doc = docMapper.parse(source("123", b -> b.field("@timestamp", "2012-02-13").field("field", "value1"), null)); - assertNotNull(doc.rootDoc().getField("_recovery_source")); - assertThat( - doc.rootDoc().getField("_recovery_source").binaryValue(), - equalTo(new BytesRef("{\"@timestamp\":\"2012-02-13\",\"field\":\"value1\"}")) - ); + if (IndexSettings.RECOVERY_USE_SYNTHETIC_SOURCE.isEnabled() == false) { + // TODO: remove this if branch when removing the 'index_recovery_use_synthetic_source' feature flag + assertNotNull(doc.rootDoc().getField("_recovery_source")); + assertThat( + doc.rootDoc().getField("_recovery_source").binaryValue(), + equalTo(new BytesRef("{\"@timestamp\":\"2012-02-13\",\"field\":\"value1\"}")) + ); + } else { + assertNull(doc.rootDoc().getField("_recovery_source")); + } } { Settings settings = Settings.builder() @@ -790,11 +816,16 @@ public void testRecoverySourceWithTimeSeriesCustom() throws IOException { MapperService mapperService = createMapperService(settings, mappings); DocumentMapper docMapper = mapperService.documentMapper(); ParsedDocument doc = docMapper.parse(source("123", b -> b.field("@timestamp", "2012-02-13").field("field", "value1"), null)); - assertNotNull(doc.rootDoc().getField("_recovery_source")); - assertThat( - doc.rootDoc().getField("_recovery_source").binaryValue(), - equalTo(new BytesRef("{\"@timestamp\":\"2012-02-13\",\"field\":\"value1\"}")) - ); + if (IndexSettings.RECOVERY_USE_SYNTHETIC_SOURCE.isEnabled() == false) { + // TODO: remove this if branch when removing the 'index_recovery_use_synthetic_source' feature flag + assertNotNull(doc.rootDoc().getField("_recovery_source")); + assertThat( + doc.rootDoc().getField("_recovery_source").binaryValue(), + equalTo(new BytesRef("{\"@timestamp\":\"2012-02-13\",\"field\":\"value1\"}")) + ); + } else { + assertNull(doc.rootDoc().getField("_recovery_source")); + } } { Settings settings = Settings.builder() From 9936a10899db4541693ac50bf9e6232bb5baf45f Mon Sep 17 00:00:00 2001 From: Salvatore Campagna Date: Thu, 30 Jan 2025 18:46:38 +0100 Subject: [PATCH 34/54] fix: lucene version --- server/src/main/java/org/elasticsearch/index/IndexVersions.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/main/java/org/elasticsearch/index/IndexVersions.java b/server/src/main/java/org/elasticsearch/index/IndexVersions.java index c46f9fbca4572..21775df4c5001 100644 --- a/server/src/main/java/org/elasticsearch/index/IndexVersions.java +++ b/server/src/main/java/org/elasticsearch/index/IndexVersions.java @@ -146,7 +146,7 @@ private static Version parseUnchecked(String version) { public static final IndexVersion SOURCE_MAPPER_MODE_ATTRIBUTE_NOOP = def(9_007_00_0, Version.LUCENE_10_0_0); public static final IndexVersion HOSTNAME_DOC_VALUES_SPARSE_INDEX = def(9_008_00_0, Version.LUCENE_10_0_0); public static final IndexVersion UPGRADE_TO_LUCENE_10_1_0 = def(9_009_00_0, Version.LUCENE_10_1_0); - public static final IndexVersion USE_SYNTHETIC_SOURCE_FOR_RECOVERY_BY_DEFAULT = def(9_010_00_0, Version.LUCENE_10_0_0); + public static final IndexVersion USE_SYNTHETIC_SOURCE_FOR_RECOVERY_BY_DEFAULT = def(9_010_00_0, Version.LUCENE_10_1_0); /* * STOP! READ THIS FIRST! No, really, * ____ _____ ___ ____ _ ____ _____ _ ____ _____ _ _ ___ ____ _____ ___ ____ ____ _____ _ From 957fd47228b768b1f353d1c162f26ca38af1dde6 Mon Sep 17 00:00:00 2001 From: Salvatore Campagna Date: Fri, 31 Jan 2025 14:37:14 +0100 Subject: [PATCH 35/54] fix: disable synthetic recovery yaml tests --- rest-api-spec/build.gradle | 2 ++ 1 file changed, 2 insertions(+) diff --git a/rest-api-spec/build.gradle b/rest-api-spec/build.gradle index 67f0b5a0714a9..261f9070422f9 100644 --- a/rest-api-spec/build.gradle +++ b/rest-api-spec/build.gradle @@ -74,4 +74,6 @@ tasks.named("yamlRestCompatTestTransform").configure ({ task -> task.skipTest("index/91_metrics_no_subobjects/Metrics object indexing with synthetic source", "_source.mode mapping attribute is no-op since 9.0.0") task.skipTest("index/91_metrics_no_subobjects/Root without subobjects with synthetic source", "_source.mode mapping attribute is no-op since 9.0.0") task.skipTest("indices.create/20_synthetic_source/synthetic_source with copy_to inside nested object", "temporary until backported") + task.skipTest("update/10_synthetic_source/keyword", "synthetic source recovery behind a feature flag and not enabled in release builds") + task.skipTest("update/10_synthetic_source/stored text", "synthetic source recovery behind a feature flag and not enabled in release builds") }) From a9ef0f59848878fea9332c13e35fafb9e53fbc04 Mon Sep 17 00:00:00 2001 From: Salvatore Campagna Date: Fri, 31 Jan 2025 14:49:10 +0100 Subject: [PATCH 36/54] fix: disable synthetic recovery yaml tests --- x-pack/plugin/logsdb/build.gradle | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/x-pack/plugin/logsdb/build.gradle b/x-pack/plugin/logsdb/build.gradle index bef07258a8e33..15570818f2f1f 100644 --- a/x-pack/plugin/logsdb/build.gradle +++ b/x-pack/plugin/logsdb/build.gradle @@ -42,3 +42,9 @@ tasks.named("javaRestTest").configure { tasks.named('yamlRestTest') { usesDefaultDistribution() } + +tasks.named("yamlRestCompatTestTransform").configure ({ task -> + task.skipTest("/60_synthetic_source_recovery/synthetic recovery for time_series index", "synthetic source recovery behind a feature flag and not enabled in release builds") + task.skipTest("/60_synthetic_source_recovery/synthetic recovery for synthetic source mode index", "synthetic source recovery behind a feature flag and not enabled in release builds") + task.skipTest("/60_synthetic_source_recovery/synthetic recovery for logsdb index", "synthetic source recovery behind a feature flag and not enabled in release builds") +}) From c140cfb1dcfb7c100f86e124ea89abdb9918b2e1 Mon Sep 17 00:00:00 2001 From: Salvatore Campagna Date: Fri, 31 Jan 2025 14:58:31 +0100 Subject: [PATCH 37/54] fix: rename task --- x-pack/plugin/logsdb/build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugin/logsdb/build.gradle b/x-pack/plugin/logsdb/build.gradle index 15570818f2f1f..00bcc51831b8f 100644 --- a/x-pack/plugin/logsdb/build.gradle +++ b/x-pack/plugin/logsdb/build.gradle @@ -43,7 +43,7 @@ tasks.named('yamlRestTest') { usesDefaultDistribution() } -tasks.named("yamlRestCompatTestTransform").configure ({ task -> +tasks.named("yamlRestTest").configure ({ task -> task.skipTest("/60_synthetic_source_recovery/synthetic recovery for time_series index", "synthetic source recovery behind a feature flag and not enabled in release builds") task.skipTest("/60_synthetic_source_recovery/synthetic recovery for synthetic source mode index", "synthetic source recovery behind a feature flag and not enabled in release builds") task.skipTest("/60_synthetic_source_recovery/synthetic recovery for logsdb index", "synthetic source recovery behind a feature flag and not enabled in release builds") From 96502e3ef2c8ae7279e3f5a7d4b4392a22f48278 Mon Sep 17 00:00:00 2001 From: Salvatore Campagna Date: Fri, 31 Jan 2025 15:13:31 +0100 Subject: [PATCH 38/54] fix: gate yaml test execution --- rest-api-spec/build.gradle | 2 -- .../rest-api-spec/test/update/100_synthetic_source.yml | 4 ++-- .../java/org/elasticsearch/index/mapper/MapperFeatures.java | 4 +++- x-pack/plugin/logsdb/build.gradle | 6 +----- .../rest-api-spec/test/60_synthetic_source_recovery.yml | 6 +++--- 5 files changed, 9 insertions(+), 13 deletions(-) diff --git a/rest-api-spec/build.gradle b/rest-api-spec/build.gradle index 261f9070422f9..67f0b5a0714a9 100644 --- a/rest-api-spec/build.gradle +++ b/rest-api-spec/build.gradle @@ -74,6 +74,4 @@ tasks.named("yamlRestCompatTestTransform").configure ({ task -> task.skipTest("index/91_metrics_no_subobjects/Metrics object indexing with synthetic source", "_source.mode mapping attribute is no-op since 9.0.0") task.skipTest("index/91_metrics_no_subobjects/Root without subobjects with synthetic source", "_source.mode mapping attribute is no-op since 9.0.0") task.skipTest("indices.create/20_synthetic_source/synthetic_source with copy_to inside nested object", "temporary until backported") - task.skipTest("update/10_synthetic_source/keyword", "synthetic source recovery behind a feature flag and not enabled in release builds") - task.skipTest("update/10_synthetic_source/stored text", "synthetic source recovery behind a feature flag and not enabled in release builds") }) diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/update/100_synthetic_source.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/update/100_synthetic_source.yml index 219bc52c4e28c..c8d8719534201 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/update/100_synthetic_source.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/update/100_synthetic_source.yml @@ -6,7 +6,7 @@ setup: --- keyword: - requires: - cluster_features: [ "mapper.synthetic_recovery_source" ] + cluster_features: [ "index.recovery.synthetic_source" ] reason: requires synthetic recovery source - do: @@ -66,7 +66,7 @@ keyword: --- stored text: - requires: - cluster_features: [ "mapper.synthetic_recovery_source" ] + cluster_features: [ "index.recovery.synthetic_source" ] reason: requires synthetic recovery source - do: diff --git a/server/src/main/java/org/elasticsearch/index/mapper/MapperFeatures.java b/server/src/main/java/org/elasticsearch/index/mapper/MapperFeatures.java index 7567fae7d73e6..4b256c86f2fca 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/MapperFeatures.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/MapperFeatures.java @@ -31,6 +31,7 @@ public class MapperFeatures implements FeatureSpecification { public static final NodeFeature SPARSE_VECTOR_STORE_SUPPORT = new NodeFeature("mapper.sparse_vector.store_support"); public static final NodeFeature SORT_FIELDS_CHECK_FOR_NESTED_OBJECT_FIX = new NodeFeature("mapper.nested.sorting_fields_check_fix"); public static final NodeFeature DYNAMIC_HANDLING_IN_COPY_TO = new NodeFeature("mapper.copy_to.dynamic_handling"); + public static final NodeFeature SYNTHETIC_SOURCE_RECOVERY = new NodeFeature("index.recovery.synthetic_source"); @Override public Set getTestFeatures() { @@ -50,7 +51,8 @@ public Set getTestFeatures() { SORT_FIELDS_CHECK_FOR_NESTED_OBJECT_FIX, DYNAMIC_HANDLING_IN_COPY_TO, SourceFieldMapper.SYNTHETIC_RECOVERY_SOURCE, - ObjectMapper.SUBOBJECTS_FALSE_MAPPING_UPDATE_FIX + ObjectMapper.SUBOBJECTS_FALSE_MAPPING_UPDATE_FIX, + SYNTHETIC_SOURCE_RECOVERY ); } } diff --git a/x-pack/plugin/logsdb/build.gradle b/x-pack/plugin/logsdb/build.gradle index 00bcc51831b8f..f8e152115ce13 100644 --- a/x-pack/plugin/logsdb/build.gradle +++ b/x-pack/plugin/logsdb/build.gradle @@ -43,8 +43,4 @@ tasks.named('yamlRestTest') { usesDefaultDistribution() } -tasks.named("yamlRestTest").configure ({ task -> - task.skipTest("/60_synthetic_source_recovery/synthetic recovery for time_series index", "synthetic source recovery behind a feature flag and not enabled in release builds") - task.skipTest("/60_synthetic_source_recovery/synthetic recovery for synthetic source mode index", "synthetic source recovery behind a feature flag and not enabled in release builds") - task.skipTest("/60_synthetic_source_recovery/synthetic recovery for logsdb index", "synthetic source recovery behind a feature flag and not enabled in release builds") -}) + diff --git a/x-pack/plugin/logsdb/src/yamlRestTest/resources/rest-api-spec/test/60_synthetic_source_recovery.yml b/x-pack/plugin/logsdb/src/yamlRestTest/resources/rest-api-spec/test/60_synthetic_source_recovery.yml index cc2216997c6d3..2269f243c241a 100644 --- a/x-pack/plugin/logsdb/src/yamlRestTest/resources/rest-api-spec/test/60_synthetic_source_recovery.yml +++ b/x-pack/plugin/logsdb/src/yamlRestTest/resources/rest-api-spec/test/60_synthetic_source_recovery.yml @@ -1,7 +1,7 @@ --- synthetic recovery for synthetic source mode index: - requires: - cluster_features: [ "mapper.synthetic_recovery_source" ] + cluster_features: [ "index.recovery.synthetic_source" ] reason: requires synthetic recovery source - do: @@ -88,7 +88,7 @@ synthetic recovery for standard index: --- synthetic recovery for logsdb index: - requires: - cluster_features: [ "mapper.synthetic_recovery_source" ] + cluster_features: [ "index.recovery.synthetic_source" ] reason: requires synthetic recovery source - do: @@ -109,7 +109,7 @@ synthetic recovery for logsdb index: --- synthetic recovery for time_series index: - requires: - cluster_features: [ "mapper.synthetic_recovery_source" ] + cluster_features: [ "index.recovery.synthetic_source" ] reason: requires synthetic recovery source - do: From fa9fd70f7e5fa384f85d8cf5e899f75ea11de8e9 Mon Sep 17 00:00:00 2001 From: Salvatore Campagna Date: Fri, 31 Jan 2025 15:16:14 +0100 Subject: [PATCH 39/54] fix: use a better feature name and description --- .../test/update/100_synthetic_source.yml | 8 ++++---- .../elasticsearch/index/mapper/MapperFeatures.java | 4 +++- .../test/60_synthetic_source_recovery.yml | 12 ++++++------ 3 files changed, 13 insertions(+), 11 deletions(-) diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/update/100_synthetic_source.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/update/100_synthetic_source.yml index c8d8719534201..e3ae82b79ab58 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/update/100_synthetic_source.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/update/100_synthetic_source.yml @@ -6,8 +6,8 @@ setup: --- keyword: - requires: - cluster_features: [ "index.recovery.synthetic_source" ] - reason: requires synthetic recovery source + cluster_features: [ "index.recovery.synthetic_source_default" ] + reason: requires synthetic recovery source by default - do: indices.create: @@ -66,8 +66,8 @@ keyword: --- stored text: - requires: - cluster_features: [ "index.recovery.synthetic_source" ] - reason: requires synthetic recovery source + cluster_features: [ "index.recovery.synthetic_source_default" ] + reason: requires synthetic recovery source by default - do: indices.create: diff --git a/server/src/main/java/org/elasticsearch/index/mapper/MapperFeatures.java b/server/src/main/java/org/elasticsearch/index/mapper/MapperFeatures.java index 4b256c86f2fca..34dcee47705e4 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/MapperFeatures.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/MapperFeatures.java @@ -31,7 +31,9 @@ public class MapperFeatures implements FeatureSpecification { public static final NodeFeature SPARSE_VECTOR_STORE_SUPPORT = new NodeFeature("mapper.sparse_vector.store_support"); public static final NodeFeature SORT_FIELDS_CHECK_FOR_NESTED_OBJECT_FIX = new NodeFeature("mapper.nested.sorting_fields_check_fix"); public static final NodeFeature DYNAMIC_HANDLING_IN_COPY_TO = new NodeFeature("mapper.copy_to.dynamic_handling"); - public static final NodeFeature SYNTHETIC_SOURCE_RECOVERY = new NodeFeature("index.recovery.synthetic_source"); + public static final NodeFeature USE_SYNTHETIC_SOURCE_FOR_RECOVERY_BY_DEFAULT = new NodeFeature( + "index.recovery.synthetic_source_default" + ); @Override public Set getTestFeatures() { diff --git a/x-pack/plugin/logsdb/src/yamlRestTest/resources/rest-api-spec/test/60_synthetic_source_recovery.yml b/x-pack/plugin/logsdb/src/yamlRestTest/resources/rest-api-spec/test/60_synthetic_source_recovery.yml index 2269f243c241a..25db4302e1ce3 100644 --- a/x-pack/plugin/logsdb/src/yamlRestTest/resources/rest-api-spec/test/60_synthetic_source_recovery.yml +++ b/x-pack/plugin/logsdb/src/yamlRestTest/resources/rest-api-spec/test/60_synthetic_source_recovery.yml @@ -1,8 +1,8 @@ --- synthetic recovery for synthetic source mode index: - requires: - cluster_features: [ "index.recovery.synthetic_source" ] - reason: requires synthetic recovery source + cluster_features: [ "index.recovery.synthetic_source_default" ] + reason: requires synthetic recovery source by default - do: indices.create: @@ -88,8 +88,8 @@ synthetic recovery for standard index: --- synthetic recovery for logsdb index: - requires: - cluster_features: [ "index.recovery.synthetic_source" ] - reason: requires synthetic recovery source + cluster_features: [ "index.recovery.synthetic_source_default" ] + reason: requires synthetic recovery source by default - do: indices.create: @@ -109,8 +109,8 @@ synthetic recovery for logsdb index: --- synthetic recovery for time_series index: - requires: - cluster_features: [ "index.recovery.synthetic_source" ] - reason: requires synthetic recovery source + cluster_features: [ "index.recovery.synthetic_source_default" ] + reason: requires synthetic recovery source by default - do: indices.create: From 502c710d98bf115b7fced411eab683386ec4a07f Mon Sep 17 00:00:00 2001 From: Salvatore Campagna Date: Fri, 31 Jan 2025 15:21:58 +0100 Subject: [PATCH 40/54] fix: push missing file --- .../index/mapper/MapperFeatures.java | 25 ++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/server/src/main/java/org/elasticsearch/index/mapper/MapperFeatures.java b/server/src/main/java/org/elasticsearch/index/mapper/MapperFeatures.java index 34dcee47705e4..13c1582f5eada 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/MapperFeatures.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/MapperFeatures.java @@ -11,6 +11,7 @@ import org.elasticsearch.features.FeatureSpecification; import org.elasticsearch.features.NodeFeature; +import org.elasticsearch.index.IndexSettings; import java.util.Set; @@ -37,6 +38,28 @@ public class MapperFeatures implements FeatureSpecification { @Override public Set getTestFeatures() { + if (IndexSettings.RECOVERY_USE_SYNTHETIC_SOURCE.isEnabled() == false) { + // NOTE: release build node features + return Set.of( + RangeFieldMapper.DATE_RANGE_INDEXING_FIX, + IgnoredSourceFieldMapper.DONT_EXPAND_DOTS_IN_IGNORED_SOURCE, + SourceFieldMapper.REMOVE_SYNTHETIC_SOURCE_ONLY_VALIDATION, + SourceFieldMapper.SOURCE_MODE_FROM_INDEX_SETTING, + IgnoredSourceFieldMapper.IGNORED_SOURCE_AS_TOP_LEVEL_METADATA_ARRAY_FIELD, + IgnoredSourceFieldMapper.ALWAYS_STORE_OBJECT_ARRAYS_IN_NESTED_OBJECTS, + MapperService.LOGSDB_DEFAULT_IGNORE_DYNAMIC_BEYOND_LIMIT, + DocumentParser.FIX_PARSING_SUBOBJECTS_FALSE_DYNAMIC_FALSE, + CONSTANT_KEYWORD_SYNTHETIC_SOURCE_WRITE_FIX, + META_FETCH_FIELDS_ERROR_CODE_CHANGED, + SPARSE_VECTOR_STORE_SUPPORT, + COUNTED_KEYWORD_SYNTHETIC_SOURCE_NATIVE_SUPPORT, + SORT_FIELDS_CHECK_FOR_NESTED_OBJECT_FIX, + DYNAMIC_HANDLING_IN_COPY_TO, + SourceFieldMapper.SYNTHETIC_RECOVERY_SOURCE, + ObjectMapper.SUBOBJECTS_FALSE_MAPPING_UPDATE_FIX + ); + } + // NOTE: snapshot build node features return Set.of( RangeFieldMapper.DATE_RANGE_INDEXING_FIX, IgnoredSourceFieldMapper.DONT_EXPAND_DOTS_IN_IGNORED_SOURCE, @@ -54,7 +77,7 @@ public Set getTestFeatures() { DYNAMIC_HANDLING_IN_COPY_TO, SourceFieldMapper.SYNTHETIC_RECOVERY_SOURCE, ObjectMapper.SUBOBJECTS_FALSE_MAPPING_UPDATE_FIX, - SYNTHETIC_SOURCE_RECOVERY + USE_SYNTHETIC_SOURCE_FOR_RECOVERY_BY_DEFAULT ); } } From c56ec165bf913a8edd00106c0f3850f2cb9c4eaa Mon Sep 17 00:00:00 2001 From: Salvatore Campagna Date: Fri, 31 Jan 2025 15:22:54 +0100 Subject: [PATCH 41/54] not: add a comment --- .../java/org/elasticsearch/index/mapper/MapperFeatures.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/main/java/org/elasticsearch/index/mapper/MapperFeatures.java b/server/src/main/java/org/elasticsearch/index/mapper/MapperFeatures.java index 13c1582f5eada..c721e3d6ec0e5 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/MapperFeatures.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/MapperFeatures.java @@ -77,7 +77,7 @@ public Set getTestFeatures() { DYNAMIC_HANDLING_IN_COPY_TO, SourceFieldMapper.SYNTHETIC_RECOVERY_SOURCE, ObjectMapper.SUBOBJECTS_FALSE_MAPPING_UPDATE_FIX, - USE_SYNTHETIC_SOURCE_FOR_RECOVERY_BY_DEFAULT + USE_SYNTHETIC_SOURCE_FOR_RECOVERY_BY_DEFAULT // NOTE: only for snapshot builds ); } } From 5c2a1b905bfc5fc5939d0bd62d02a8b313625bcc Mon Sep 17 00:00:00 2001 From: Salvatore Campagna Date: Fri, 31 Jan 2025 15:33:25 +0100 Subject: [PATCH 42/54] fix: refactor feature selection logic --- .../index/mapper/MapperFeatures.java | 68 ++++++++----------- 1 file changed, 28 insertions(+), 40 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/index/mapper/MapperFeatures.java b/server/src/main/java/org/elasticsearch/index/mapper/MapperFeatures.java index c721e3d6ec0e5..00e4efbd88ef8 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/MapperFeatures.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/MapperFeatures.java @@ -13,6 +13,7 @@ import org.elasticsearch.features.NodeFeature; import org.elasticsearch.index.IndexSettings; +import java.util.HashSet; import java.util.Set; /** @@ -32,52 +33,39 @@ public class MapperFeatures implements FeatureSpecification { public static final NodeFeature SPARSE_VECTOR_STORE_SUPPORT = new NodeFeature("mapper.sparse_vector.store_support"); public static final NodeFeature SORT_FIELDS_CHECK_FOR_NESTED_OBJECT_FIX = new NodeFeature("mapper.nested.sorting_fields_check_fix"); public static final NodeFeature DYNAMIC_HANDLING_IN_COPY_TO = new NodeFeature("mapper.copy_to.dynamic_handling"); + public static final NodeFeature USE_SYNTHETIC_SOURCE_FOR_RECOVERY_BY_DEFAULT = new NodeFeature( "index.recovery.synthetic_source_default" ); + private static final Set DEFAULT_TEST_FEATURES = Set.of( + RangeFieldMapper.DATE_RANGE_INDEXING_FIX, + IgnoredSourceFieldMapper.DONT_EXPAND_DOTS_IN_IGNORED_SOURCE, + SourceFieldMapper.REMOVE_SYNTHETIC_SOURCE_ONLY_VALIDATION, + SourceFieldMapper.SOURCE_MODE_FROM_INDEX_SETTING, + IgnoredSourceFieldMapper.IGNORED_SOURCE_AS_TOP_LEVEL_METADATA_ARRAY_FIELD, + IgnoredSourceFieldMapper.ALWAYS_STORE_OBJECT_ARRAYS_IN_NESTED_OBJECTS, + MapperService.LOGSDB_DEFAULT_IGNORE_DYNAMIC_BEYOND_LIMIT, + DocumentParser.FIX_PARSING_SUBOBJECTS_FALSE_DYNAMIC_FALSE, + CONSTANT_KEYWORD_SYNTHETIC_SOURCE_WRITE_FIX, + META_FETCH_FIELDS_ERROR_CODE_CHANGED, + SPARSE_VECTOR_STORE_SUPPORT, + COUNTED_KEYWORD_SYNTHETIC_SOURCE_NATIVE_SUPPORT, + SORT_FIELDS_CHECK_FOR_NESTED_OBJECT_FIX, + DYNAMIC_HANDLING_IN_COPY_TO, + SourceFieldMapper.SYNTHETIC_RECOVERY_SOURCE, + ObjectMapper.SUBOBJECTS_FALSE_MAPPING_UPDATE_FIX + ); + + private static final Set ADDITIONAL_TEST_FEATURES = Set.of(USE_SYNTHETIC_SOURCE_FOR_RECOVERY_BY_DEFAULT); + @Override public Set getTestFeatures() { - if (IndexSettings.RECOVERY_USE_SYNTHETIC_SOURCE.isEnabled() == false) { - // NOTE: release build node features - return Set.of( - RangeFieldMapper.DATE_RANGE_INDEXING_FIX, - IgnoredSourceFieldMapper.DONT_EXPAND_DOTS_IN_IGNORED_SOURCE, - SourceFieldMapper.REMOVE_SYNTHETIC_SOURCE_ONLY_VALIDATION, - SourceFieldMapper.SOURCE_MODE_FROM_INDEX_SETTING, - IgnoredSourceFieldMapper.IGNORED_SOURCE_AS_TOP_LEVEL_METADATA_ARRAY_FIELD, - IgnoredSourceFieldMapper.ALWAYS_STORE_OBJECT_ARRAYS_IN_NESTED_OBJECTS, - MapperService.LOGSDB_DEFAULT_IGNORE_DYNAMIC_BEYOND_LIMIT, - DocumentParser.FIX_PARSING_SUBOBJECTS_FALSE_DYNAMIC_FALSE, - CONSTANT_KEYWORD_SYNTHETIC_SOURCE_WRITE_FIX, - META_FETCH_FIELDS_ERROR_CODE_CHANGED, - SPARSE_VECTOR_STORE_SUPPORT, - COUNTED_KEYWORD_SYNTHETIC_SOURCE_NATIVE_SUPPORT, - SORT_FIELDS_CHECK_FOR_NESTED_OBJECT_FIX, - DYNAMIC_HANDLING_IN_COPY_TO, - SourceFieldMapper.SYNTHETIC_RECOVERY_SOURCE, - ObjectMapper.SUBOBJECTS_FALSE_MAPPING_UPDATE_FIX - ); + if (IndexSettings.RECOVERY_USE_SYNTHETIC_SOURCE.isEnabled()) { + final Set features = new HashSet<>(DEFAULT_TEST_FEATURES); + features.addAll(ADDITIONAL_TEST_FEATURES); + return features; } - // NOTE: snapshot build node features - return Set.of( - RangeFieldMapper.DATE_RANGE_INDEXING_FIX, - IgnoredSourceFieldMapper.DONT_EXPAND_DOTS_IN_IGNORED_SOURCE, - SourceFieldMapper.REMOVE_SYNTHETIC_SOURCE_ONLY_VALIDATION, - SourceFieldMapper.SOURCE_MODE_FROM_INDEX_SETTING, - IgnoredSourceFieldMapper.IGNORED_SOURCE_AS_TOP_LEVEL_METADATA_ARRAY_FIELD, - IgnoredSourceFieldMapper.ALWAYS_STORE_OBJECT_ARRAYS_IN_NESTED_OBJECTS, - MapperService.LOGSDB_DEFAULT_IGNORE_DYNAMIC_BEYOND_LIMIT, - DocumentParser.FIX_PARSING_SUBOBJECTS_FALSE_DYNAMIC_FALSE, - CONSTANT_KEYWORD_SYNTHETIC_SOURCE_WRITE_FIX, - META_FETCH_FIELDS_ERROR_CODE_CHANGED, - SPARSE_VECTOR_STORE_SUPPORT, - COUNTED_KEYWORD_SYNTHETIC_SOURCE_NATIVE_SUPPORT, - SORT_FIELDS_CHECK_FOR_NESTED_OBJECT_FIX, - DYNAMIC_HANDLING_IN_COPY_TO, - SourceFieldMapper.SYNTHETIC_RECOVERY_SOURCE, - ObjectMapper.SUBOBJECTS_FALSE_MAPPING_UPDATE_FIX, - USE_SYNTHETIC_SOURCE_FOR_RECOVERY_BY_DEFAULT // NOTE: only for snapshot builds - ); + return DEFAULT_TEST_FEATURES; } } From d865ff54721128c6f5cc40539b5fd08690931091 Mon Sep 17 00:00:00 2001 From: Salvatore Campagna Date: Fri, 31 Jan 2025 15:35:40 +0100 Subject: [PATCH 43/54] nit: avoid creating another static set --- .../java/org/elasticsearch/index/mapper/MapperFeatures.java | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/index/mapper/MapperFeatures.java b/server/src/main/java/org/elasticsearch/index/mapper/MapperFeatures.java index 00e4efbd88ef8..c0ffd4386ade0 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/MapperFeatures.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/MapperFeatures.java @@ -57,13 +57,11 @@ public class MapperFeatures implements FeatureSpecification { ObjectMapper.SUBOBJECTS_FALSE_MAPPING_UPDATE_FIX ); - private static final Set ADDITIONAL_TEST_FEATURES = Set.of(USE_SYNTHETIC_SOURCE_FOR_RECOVERY_BY_DEFAULT); - @Override public Set getTestFeatures() { if (IndexSettings.RECOVERY_USE_SYNTHETIC_SOURCE.isEnabled()) { final Set features = new HashSet<>(DEFAULT_TEST_FEATURES); - features.addAll(ADDITIONAL_TEST_FEATURES); + features.add(USE_SYNTHETIC_SOURCE_FOR_RECOVERY_BY_DEFAULT); return features; } return DEFAULT_TEST_FEATURES; From 14997b78febe9d19f4eedec64b0ae76b1d6b0f01 Mon Sep 17 00:00:00 2001 From: Salvatore Campagna Date: Fri, 31 Jan 2025 16:15:33 +0100 Subject: [PATCH 44/54] fix: exclude tests from release build --- rest-api-spec/build.gradle | 2 + .../index/mapper/MapperFeatures.java | 49 +++++++------------ x-pack/plugin/logsdb/build.gradle | 8 +++ 3 files changed, 28 insertions(+), 31 deletions(-) diff --git a/rest-api-spec/build.gradle b/rest-api-spec/build.gradle index 67f0b5a0714a9..261f9070422f9 100644 --- a/rest-api-spec/build.gradle +++ b/rest-api-spec/build.gradle @@ -74,4 +74,6 @@ tasks.named("yamlRestCompatTestTransform").configure ({ task -> task.skipTest("index/91_metrics_no_subobjects/Metrics object indexing with synthetic source", "_source.mode mapping attribute is no-op since 9.0.0") task.skipTest("index/91_metrics_no_subobjects/Root without subobjects with synthetic source", "_source.mode mapping attribute is no-op since 9.0.0") task.skipTest("indices.create/20_synthetic_source/synthetic_source with copy_to inside nested object", "temporary until backported") + task.skipTest("update/10_synthetic_source/keyword", "synthetic source recovery behind a feature flag and not enabled in release builds") + task.skipTest("update/10_synthetic_source/stored text", "synthetic source recovery behind a feature flag and not enabled in release builds") }) diff --git a/server/src/main/java/org/elasticsearch/index/mapper/MapperFeatures.java b/server/src/main/java/org/elasticsearch/index/mapper/MapperFeatures.java index c0ffd4386ade0..7567fae7d73e6 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/MapperFeatures.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/MapperFeatures.java @@ -11,9 +11,7 @@ import org.elasticsearch.features.FeatureSpecification; import org.elasticsearch.features.NodeFeature; -import org.elasticsearch.index.IndexSettings; -import java.util.HashSet; import java.util.Set; /** @@ -34,36 +32,25 @@ public class MapperFeatures implements FeatureSpecification { public static final NodeFeature SORT_FIELDS_CHECK_FOR_NESTED_OBJECT_FIX = new NodeFeature("mapper.nested.sorting_fields_check_fix"); public static final NodeFeature DYNAMIC_HANDLING_IN_COPY_TO = new NodeFeature("mapper.copy_to.dynamic_handling"); - public static final NodeFeature USE_SYNTHETIC_SOURCE_FOR_RECOVERY_BY_DEFAULT = new NodeFeature( - "index.recovery.synthetic_source_default" - ); - - private static final Set DEFAULT_TEST_FEATURES = Set.of( - RangeFieldMapper.DATE_RANGE_INDEXING_FIX, - IgnoredSourceFieldMapper.DONT_EXPAND_DOTS_IN_IGNORED_SOURCE, - SourceFieldMapper.REMOVE_SYNTHETIC_SOURCE_ONLY_VALIDATION, - SourceFieldMapper.SOURCE_MODE_FROM_INDEX_SETTING, - IgnoredSourceFieldMapper.IGNORED_SOURCE_AS_TOP_LEVEL_METADATA_ARRAY_FIELD, - IgnoredSourceFieldMapper.ALWAYS_STORE_OBJECT_ARRAYS_IN_NESTED_OBJECTS, - MapperService.LOGSDB_DEFAULT_IGNORE_DYNAMIC_BEYOND_LIMIT, - DocumentParser.FIX_PARSING_SUBOBJECTS_FALSE_DYNAMIC_FALSE, - CONSTANT_KEYWORD_SYNTHETIC_SOURCE_WRITE_FIX, - META_FETCH_FIELDS_ERROR_CODE_CHANGED, - SPARSE_VECTOR_STORE_SUPPORT, - COUNTED_KEYWORD_SYNTHETIC_SOURCE_NATIVE_SUPPORT, - SORT_FIELDS_CHECK_FOR_NESTED_OBJECT_FIX, - DYNAMIC_HANDLING_IN_COPY_TO, - SourceFieldMapper.SYNTHETIC_RECOVERY_SOURCE, - ObjectMapper.SUBOBJECTS_FALSE_MAPPING_UPDATE_FIX - ); - @Override public Set getTestFeatures() { - if (IndexSettings.RECOVERY_USE_SYNTHETIC_SOURCE.isEnabled()) { - final Set features = new HashSet<>(DEFAULT_TEST_FEATURES); - features.add(USE_SYNTHETIC_SOURCE_FOR_RECOVERY_BY_DEFAULT); - return features; - } - return DEFAULT_TEST_FEATURES; + return Set.of( + RangeFieldMapper.DATE_RANGE_INDEXING_FIX, + IgnoredSourceFieldMapper.DONT_EXPAND_DOTS_IN_IGNORED_SOURCE, + SourceFieldMapper.REMOVE_SYNTHETIC_SOURCE_ONLY_VALIDATION, + SourceFieldMapper.SOURCE_MODE_FROM_INDEX_SETTING, + IgnoredSourceFieldMapper.IGNORED_SOURCE_AS_TOP_LEVEL_METADATA_ARRAY_FIELD, + IgnoredSourceFieldMapper.ALWAYS_STORE_OBJECT_ARRAYS_IN_NESTED_OBJECTS, + MapperService.LOGSDB_DEFAULT_IGNORE_DYNAMIC_BEYOND_LIMIT, + DocumentParser.FIX_PARSING_SUBOBJECTS_FALSE_DYNAMIC_FALSE, + CONSTANT_KEYWORD_SYNTHETIC_SOURCE_WRITE_FIX, + META_FETCH_FIELDS_ERROR_CODE_CHANGED, + SPARSE_VECTOR_STORE_SUPPORT, + COUNTED_KEYWORD_SYNTHETIC_SOURCE_NATIVE_SUPPORT, + SORT_FIELDS_CHECK_FOR_NESTED_OBJECT_FIX, + DYNAMIC_HANDLING_IN_COPY_TO, + SourceFieldMapper.SYNTHETIC_RECOVERY_SOURCE, + ObjectMapper.SUBOBJECTS_FALSE_MAPPING_UPDATE_FIX + ); } } diff --git a/x-pack/plugin/logsdb/build.gradle b/x-pack/plugin/logsdb/build.gradle index f8e152115ce13..c4a088084734a 100644 --- a/x-pack/plugin/logsdb/build.gradle +++ b/x-pack/plugin/logsdb/build.gradle @@ -43,4 +43,12 @@ tasks.named('yamlRestTest') { usesDefaultDistribution() } +tasks.named("yamlRestTest") { + if (buildParams.isSnapshotBuild() == false) { + systemProperty 'tests.rest.blacklist', [ + "resources/rest-api-spec/test/60_synthetic_source_recovery.yml" + ].join(',') + } +} + From ad7d7421db588d2897a851737ccc374ed7e9187b Mon Sep 17 00:00:00 2001 From: Salvatore Campagna Date: Fri, 31 Jan 2025 16:40:07 +0100 Subject: [PATCH 45/54] fix: exlude from rest compatibility tests --- rest-api-spec/build.gradle | 3 +++ 1 file changed, 3 insertions(+) diff --git a/rest-api-spec/build.gradle b/rest-api-spec/build.gradle index 261f9070422f9..629f86ce3efef 100644 --- a/rest-api-spec/build.gradle +++ b/rest-api-spec/build.gradle @@ -74,6 +74,9 @@ tasks.named("yamlRestCompatTestTransform").configure ({ task -> task.skipTest("index/91_metrics_no_subobjects/Metrics object indexing with synthetic source", "_source.mode mapping attribute is no-op since 9.0.0") task.skipTest("index/91_metrics_no_subobjects/Root without subobjects with synthetic source", "_source.mode mapping attribute is no-op since 9.0.0") task.skipTest("indices.create/20_synthetic_source/synthetic_source with copy_to inside nested object", "temporary until backported") +}) + +tasks.named("yamlRestCompatTest").configure ({ task -> task.skipTest("update/10_synthetic_source/keyword", "synthetic source recovery behind a feature flag and not enabled in release builds") task.skipTest("update/10_synthetic_source/stored text", "synthetic source recovery behind a feature flag and not enabled in release builds") }) From e72b3a8ca0237242493369687a334cb6277c2bef Mon Sep 17 00:00:00 2001 From: Salvatore Campagna Date: Fri, 31 Jan 2025 16:52:44 +0100 Subject: [PATCH 46/54] fix: try to disable tests --- muted-tests.yml | 4 ++++ rest-api-spec/build.gradle | 5 ----- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/muted-tests.yml b/muted-tests.yml index a326f8d9eac4f..88a0683b91551 100644 --- a/muted-tests.yml +++ b/muted-tests.yml @@ -143,6 +143,10 @@ tests: - class: org.elasticsearch.datastreams.DataStreamsClientYamlTestSuiteIT method: test {p0=data_stream/120_data_streams_stats/Multiple data stream} issue: https://github.com/elastic/elasticsearch/issues/118217 +- class: org.elasticsearch.datastreams.DataStreamsClientYamlTestSuiteIT + method: test {p0=update/100_synthetic_source/keyword} +- class: org.elasticsearch.datastreams.DataStreamsClientYamlTestSuiteIT + method: test {p0=update/100_synthetic_source/stored text} - class: org.elasticsearch.action.search.SearchQueryThenFetchAsyncActionTests method: testBottomFieldSort issue: https://github.com/elastic/elasticsearch/issues/118214 diff --git a/rest-api-spec/build.gradle b/rest-api-spec/build.gradle index 629f86ce3efef..67f0b5a0714a9 100644 --- a/rest-api-spec/build.gradle +++ b/rest-api-spec/build.gradle @@ -75,8 +75,3 @@ tasks.named("yamlRestCompatTestTransform").configure ({ task -> task.skipTest("index/91_metrics_no_subobjects/Root without subobjects with synthetic source", "_source.mode mapping attribute is no-op since 9.0.0") task.skipTest("indices.create/20_synthetic_source/synthetic_source with copy_to inside nested object", "temporary until backported") }) - -tasks.named("yamlRestCompatTest").configure ({ task -> - task.skipTest("update/10_synthetic_source/keyword", "synthetic source recovery behind a feature flag and not enabled in release builds") - task.skipTest("update/10_synthetic_source/stored text", "synthetic source recovery behind a feature flag and not enabled in release builds") -}) From 8fe2f3e8c27eee450cb0d3c83e12da28563e8f0c Mon Sep 17 00:00:00 2001 From: Salvatore Campagna Date: Fri, 31 Jan 2025 16:54:49 +0100 Subject: [PATCH 47/54] fix: use correct class --- muted-tests.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/muted-tests.yml b/muted-tests.yml index 88a0683b91551..ab77f14885370 100644 --- a/muted-tests.yml +++ b/muted-tests.yml @@ -143,9 +143,9 @@ tests: - class: org.elasticsearch.datastreams.DataStreamsClientYamlTestSuiteIT method: test {p0=data_stream/120_data_streams_stats/Multiple data stream} issue: https://github.com/elastic/elasticsearch/issues/118217 -- class: org.elasticsearch.datastreams.DataStreamsClientYamlTestSuiteIT +- class: org.elasticsearch.test.rest.ClientYamlTestSuiteIT method: test {p0=update/100_synthetic_source/keyword} -- class: org.elasticsearch.datastreams.DataStreamsClientYamlTestSuiteIT +- class: org.elasticsearch.test.rest.ClientYamlTestSuiteIT method: test {p0=update/100_synthetic_source/stored text} - class: org.elasticsearch.action.search.SearchQueryThenFetchAsyncActionTests method: testBottomFieldSort From fc8d4308c14594806f2619182011a2185ec4c896 Mon Sep 17 00:00:00 2001 From: Salvatore Campagna Date: Fri, 31 Jan 2025 17:24:36 +0100 Subject: [PATCH 48/54] fix: restore previous feature name --- .../test/update/100_synthetic_source.yml | 8 ++++---- .../test/60_synthetic_source_recovery.yml | 12 ++++++------ 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/update/100_synthetic_source.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/update/100_synthetic_source.yml index e3ae82b79ab58..219bc52c4e28c 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/update/100_synthetic_source.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/update/100_synthetic_source.yml @@ -6,8 +6,8 @@ setup: --- keyword: - requires: - cluster_features: [ "index.recovery.synthetic_source_default" ] - reason: requires synthetic recovery source by default + cluster_features: [ "mapper.synthetic_recovery_source" ] + reason: requires synthetic recovery source - do: indices.create: @@ -66,8 +66,8 @@ keyword: --- stored text: - requires: - cluster_features: [ "index.recovery.synthetic_source_default" ] - reason: requires synthetic recovery source by default + cluster_features: [ "mapper.synthetic_recovery_source" ] + reason: requires synthetic recovery source - do: indices.create: diff --git a/x-pack/plugin/logsdb/src/yamlRestTest/resources/rest-api-spec/test/60_synthetic_source_recovery.yml b/x-pack/plugin/logsdb/src/yamlRestTest/resources/rest-api-spec/test/60_synthetic_source_recovery.yml index 25db4302e1ce3..cc2216997c6d3 100644 --- a/x-pack/plugin/logsdb/src/yamlRestTest/resources/rest-api-spec/test/60_synthetic_source_recovery.yml +++ b/x-pack/plugin/logsdb/src/yamlRestTest/resources/rest-api-spec/test/60_synthetic_source_recovery.yml @@ -1,8 +1,8 @@ --- synthetic recovery for synthetic source mode index: - requires: - cluster_features: [ "index.recovery.synthetic_source_default" ] - reason: requires synthetic recovery source by default + cluster_features: [ "mapper.synthetic_recovery_source" ] + reason: requires synthetic recovery source - do: indices.create: @@ -88,8 +88,8 @@ synthetic recovery for standard index: --- synthetic recovery for logsdb index: - requires: - cluster_features: [ "index.recovery.synthetic_source_default" ] - reason: requires synthetic recovery source by default + cluster_features: [ "mapper.synthetic_recovery_source" ] + reason: requires synthetic recovery source - do: indices.create: @@ -109,8 +109,8 @@ synthetic recovery for logsdb index: --- synthetic recovery for time_series index: - requires: - cluster_features: [ "index.recovery.synthetic_source_default" ] - reason: requires synthetic recovery source by default + cluster_features: [ "mapper.synthetic_recovery_source" ] + reason: requires synthetic recovery source - do: indices.create: From e70a02b65b63032bd8d991a631d323f7d36597fa Mon Sep 17 00:00:00 2001 From: Salvatore Campagna Date: Fri, 31 Jan 2025 17:27:11 +0100 Subject: [PATCH 49/54] fix: test patch --- x-pack/plugin/logsdb/build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugin/logsdb/build.gradle b/x-pack/plugin/logsdb/build.gradle index c4a088084734a..917f8207d2d29 100644 --- a/x-pack/plugin/logsdb/build.gradle +++ b/x-pack/plugin/logsdb/build.gradle @@ -46,7 +46,7 @@ tasks.named('yamlRestTest') { tasks.named("yamlRestTest") { if (buildParams.isSnapshotBuild() == false) { systemProperty 'tests.rest.blacklist', [ - "resources/rest-api-spec/test/60_synthetic_source_recovery.yml" + "60_synthetic_source_recovery/*" ].join(',') } } From 09e523d741a6fef8bfaffecdac74515c1fcf0378 Mon Sep 17 00:00:00 2001 From: Salvatore Campagna Date: Fri, 31 Jan 2025 17:29:49 +0100 Subject: [PATCH 50/54] fix: add TODO to muted-tests --- muted-tests.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/muted-tests.yml b/muted-tests.yml index ab77f14885370..f13d7ba026682 100644 --- a/muted-tests.yml +++ b/muted-tests.yml @@ -143,8 +143,10 @@ tests: - class: org.elasticsearch.datastreams.DataStreamsClientYamlTestSuiteIT method: test {p0=data_stream/120_data_streams_stats/Multiple data stream} issue: https://github.com/elastic/elasticsearch/issues/118217 +# TODO: re-enabled after backport of https://github.com/elastic/elasticsearch/pull/119110 - class: org.elasticsearch.test.rest.ClientYamlTestSuiteIT method: test {p0=update/100_synthetic_source/keyword} + # TODO: re-enabled after backport of https://github.com/elastic/elasticsearch/pull/119110 - class: org.elasticsearch.test.rest.ClientYamlTestSuiteIT method: test {p0=update/100_synthetic_source/stored text} - class: org.elasticsearch.action.search.SearchQueryThenFetchAsyncActionTests From a7f0e9c86bab95ddbda9a15d56e09e73fa82e47d Mon Sep 17 00:00:00 2001 From: Salvatore Campagna Date: Fri, 31 Jan 2025 17:31:52 +0100 Subject: [PATCH 51/54] fix: add TODO to muted-tests --- muted-tests.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/muted-tests.yml b/muted-tests.yml index f13d7ba026682..4a0ca5fdf681b 100644 --- a/muted-tests.yml +++ b/muted-tests.yml @@ -143,10 +143,10 @@ tests: - class: org.elasticsearch.datastreams.DataStreamsClientYamlTestSuiteIT method: test {p0=data_stream/120_data_streams_stats/Multiple data stream} issue: https://github.com/elastic/elasticsearch/issues/118217 -# TODO: re-enabled after backport of https://github.com/elastic/elasticsearch/pull/119110 + # TODO: re-enable after backporting https://github.com/elastic/elasticsearch/pull/119110 - class: org.elasticsearch.test.rest.ClientYamlTestSuiteIT method: test {p0=update/100_synthetic_source/keyword} - # TODO: re-enabled after backport of https://github.com/elastic/elasticsearch/pull/119110 + # TODO: re-enable after backporting https://github.com/elastic/elasticsearch/pull/119110 - class: org.elasticsearch.test.rest.ClientYamlTestSuiteIT method: test {p0=update/100_synthetic_source/stored text} - class: org.elasticsearch.action.search.SearchQueryThenFetchAsyncActionTests From a532297938b0e1febdb1531640cfc946b3d367e1 Mon Sep 17 00:00:00 2001 From: Salvatore Campagna Date: Mon, 3 Feb 2025 11:51:42 +0100 Subject: [PATCH 52/54] fix: yaml instead of p0 --- muted-tests.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/muted-tests.yml b/muted-tests.yml index 49338a4b8015d..00e2a24d5c50f 100644 --- a/muted-tests.yml +++ b/muted-tests.yml @@ -145,10 +145,10 @@ tests: issue: https://github.com/elastic/elasticsearch/issues/118217 # TODO: re-enable after backporting https://github.com/elastic/elasticsearch/pull/119110 - class: org.elasticsearch.test.rest.ClientYamlTestSuiteIT - method: test {p0=update/100_synthetic_source/keyword} + method: test {yaml=update/100_synthetic_source/keyword} # TODO: re-enable after backporting https://github.com/elastic/elasticsearch/pull/119110 - class: org.elasticsearch.test.rest.ClientYamlTestSuiteIT - method: test {p0=update/100_synthetic_source/stored text} + method: test {yaml=update/100_synthetic_source/stored text} - class: org.elasticsearch.xpack.searchablesnapshots.RetrySearchIntegTests method: testSearcherId issue: https://github.com/elastic/elasticsearch/issues/118374 From 6d221e3b5a0576718039dc83725ea5642fcc6e4b Mon Sep 17 00:00:00 2001 From: Salvatore Campagna Date: Mon, 3 Feb 2025 13:34:40 +0100 Subject: [PATCH 53/54] fix: missing backport version --- server/src/main/java/org/elasticsearch/index/IndexVersions.java | 1 + 1 file changed, 1 insertion(+) diff --git a/server/src/main/java/org/elasticsearch/index/IndexVersions.java b/server/src/main/java/org/elasticsearch/index/IndexVersions.java index 9dae292bd40df..821eab8f49913 100644 --- a/server/src/main/java/org/elasticsearch/index/IndexVersions.java +++ b/server/src/main/java/org/elasticsearch/index/IndexVersions.java @@ -135,6 +135,7 @@ private static Version parseUnchecked(String version) { public static final IndexVersion UPGRADE_TO_LUCENE_9_12_1 = def(8_523_0_00, parseUnchecked("9.12.1")); public static final IndexVersion INFERENCE_METADATA_FIELDS_BACKPORT = def(8_524_0_00, parseUnchecked("9.12.1")); public static final IndexVersion LOGSB_OPTIONAL_SORTING_ON_HOST_NAME_BACKPORT = def(8_525_0_00, parseUnchecked("9.12.1")); + public static final IndexVersion USE_SYNTHETIC_SOURCE_FOR_RECOVERY_BY_DEFAULT_BACKPORT = def(8_526_0_00, parseUnchecked("9.12.1")); public static final IndexVersion UPGRADE_TO_LUCENE_10_0_0 = def(9_000_0_00, Version.LUCENE_10_0_0); public static final IndexVersion LOGSDB_DEFAULT_IGNORE_DYNAMIC_BEYOND_LIMIT = def(9_001_0_00, Version.LUCENE_10_0_0); public static final IndexVersion TIME_BASED_K_ORDERED_DOC_ID = def(9_002_0_00, Version.LUCENE_10_0_0); From 98657ed60fbad9fcf7ca0e6750c7a1d9edcff9b3 Mon Sep 17 00:00:00 2001 From: Salvatore Campagna Date: Mon, 3 Feb 2025 14:57:39 +0100 Subject: [PATCH 54/54] fix: try to exclude tests --- qa/smoke-test-multinode/build.gradle | 2 ++ x-pack/qa/core-rest-tests-with-security/build.gradle | 4 +++- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/qa/smoke-test-multinode/build.gradle b/qa/smoke-test-multinode/build.gradle index cc71a99562eb5..14e102025cac7 100644 --- a/qa/smoke-test-multinode/build.gradle +++ b/qa/smoke-test-multinode/build.gradle @@ -28,5 +28,7 @@ tasks.named("yamlRestTest").configure { 'cat.templates/10_basic/No templates', 'cat.templates/10_basic/Sort templates', 'cat.templates/10_basic/Multiple template', + 'update/100_synthetic_source/keyword', + 'update/100_synthetic_source/stored text' ].join(',') } diff --git a/x-pack/qa/core-rest-tests-with-security/build.gradle b/x-pack/qa/core-rest-tests-with-security/build.gradle index 65f2282014dc4..a3f50089d5429 100644 --- a/x-pack/qa/core-rest-tests-with-security/build.gradle +++ b/x-pack/qa/core-rest-tests-with-security/build.gradle @@ -31,7 +31,9 @@ restResources { tasks.named("yamlRestTest").configure { ArrayList blacklist = [ 'index/10_with_id/Index with ID', - 'indices.get_alias/10_basic/Get alias against closed indices' + 'indices.get_alias/10_basic/Get alias against closed indices', + 'update/100_synthetic_source/keyword', + 'update/100_synthetic_source/stored text' ]; if (buildParams.isSnapshotBuild() == false) { blacklist += [