Skip to content

Commit 631345f

Browse files
dnhatndnhatn
authored
Adjust index version for deprecating source mode (#117183)
There was a bug in the version-checking logic for emitting deprecation warnings for source.mode in mappings.
1 parent 4e1807f commit 631345f

File tree

14 files changed

+133
-24
lines changed

14 files changed

+133
-24
lines changed

‎modules/data-streams/src/test/java/org/elasticsearch/datastreams/mapper/DataStreamTimestampFieldMapperTests.java‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -48,7 +48,8 @@ protected void registerParameters(ParameterChecker checker) throws IOException {
4848
checker.registerConflictCheck(
4949
"enabled",
5050
timestampMapping(true, b -> b.startObject("@timestamp").field("type", "date").endObject()),
51-
timestampMapping(false, b -> b.startObject("@timestamp").field("type", "date").endObject())
51+
timestampMapping(false, b -> b.startObject("@timestamp").field("type", "date").endObject()),
52+
dm -> {}
5253
);
5354
checker.registerUpdateCheck(
5455
timestampMapping(false, b -> b.startObject("@timestamp").field("type", "date").endObject()),

‎qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/IndexingIT.java‎

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@
2020
import org.elasticsearch.common.xcontent.support.XContentMapValues;
2121
import org.elasticsearch.core.UpdateForV9;
2222
import org.elasticsearch.index.mapper.DateFieldMapper;
23+
import org.elasticsearch.index.mapper.SourceFieldMapper;
2324
import org.elasticsearch.index.mapper.TimeSeriesIdFieldMapper;
2425
import org.elasticsearch.test.ListMatcher;
2526
import org.elasticsearch.xcontent.XContentBuilder;
@@ -417,9 +418,15 @@ public void testSyntheticSource() throws IOException {
417418
if (isOldCluster()) {
418419
Request createIndex = new Request("PUT", "/synthetic");
419420
XContentBuilder indexSpec = XContentBuilder.builder(XContentType.JSON.xContent()).startObject();
421+
boolean useIndexSetting = SourceFieldMapper.onOrAfterDeprecateModeVersion(getOldClusterIndexVersion());
422+
if (useIndexSetting) {
423+
indexSpec.startObject("settings").field("index.mapping.source.mode", "synthetic").endObject();
424+
}
420425
indexSpec.startObject("mappings");
421426
{
422-
indexSpec.startObject("_source").field("mode", "synthetic").endObject();
427+
if (useIndexSetting == false) {
428+
indexSpec.startObject("_source").field("mode", "synthetic").endObject();
429+
}
423430
indexSpec.startObject("properties").startObject("kwd").field("type", "keyword").endObject().endObject();
424431
}
425432
indexSpec.endObject();

‎server/src/main/java/org/elasticsearch/index/mapper/SourceFieldMapper.java‎

Lines changed: 10 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,7 @@
2626
import org.elasticsearch.features.NodeFeature;
2727
import org.elasticsearch.index.IndexMode;
2828
import org.elasticsearch.index.IndexSettings;
29+
import org.elasticsearch.index.IndexVersion;
2930
import org.elasticsearch.index.IndexVersions;
3031
import org.elasticsearch.index.query.QueryShardException;
3132
import org.elasticsearch.index.query.SearchExecutionContext;
@@ -297,7 +298,7 @@ private static SourceFieldMapper resolveStaticInstance(final Mode sourceMode) {
297298
if (indexMode == IndexMode.STANDARD && settingSourceMode == Mode.STORED) {
298299
return DEFAULT;
299300
}
300-
if (c.indexVersionCreated().onOrAfter(IndexVersions.DEPRECATE_SOURCE_MODE_MAPPER)) {
301+
if (onOrAfterDeprecateModeVersion(c.indexVersionCreated())) {
301302
return resolveStaticInstance(settingSourceMode);
302303
} else {
303304
return new SourceFieldMapper(settingSourceMode, Explicit.IMPLICIT_TRUE, Strings.EMPTY_ARRAY, Strings.EMPTY_ARRAY, true);
@@ -307,14 +308,14 @@ private static SourceFieldMapper resolveStaticInstance(final Mode sourceMode) {
307308
c.getIndexSettings().getMode(),
308309
c.getSettings(),
309310
c.indexVersionCreated().onOrAfter(IndexVersions.SOURCE_MAPPER_LOSSY_PARAMS_CHECK),
310-
c.indexVersionCreated().before(IndexVersions.DEPRECATE_SOURCE_MODE_MAPPER)
311+
onOrAfterDeprecateModeVersion(c.indexVersionCreated()) == false
311312
)
312313
) {
313314
@Override
314315
public MetadataFieldMapper.Builder parse(String name, Map<String, Object> node, MappingParserContext parserContext)
315316
throws MapperParsingException {
316317
assert name.equals(SourceFieldMapper.NAME) : name;
317-
if (parserContext.indexVersionCreated().after(IndexVersions.DEPRECATE_SOURCE_MODE_MAPPER) && node.containsKey("mode")) {
318+
if (onOrAfterDeprecateModeVersion(parserContext.indexVersionCreated()) && node.containsKey("mode")) {
318319
deprecationLogger.critical(DeprecationCategory.MAPPINGS, "mapping_source_mode", SourceFieldMapper.DEPRECATION_WARNING);
319320
}
320321
return super.parse(name, node, parserContext);
@@ -481,4 +482,10 @@ public boolean isDisabled() {
481482
public boolean isStored() {
482483
return mode == null || mode == Mode.STORED;
483484
}
485+
486+
public static boolean onOrAfterDeprecateModeVersion(IndexVersion version) {
487+
return version.onOrAfter(IndexVersions.DEPRECATE_SOURCE_MODE_MAPPER);
488+
// Adjust versions after backporting.
489+
// || version.between(IndexVersions.BACKPORT_DEPRECATE_SOURCE_MODE_MAPPER, IndexVersions.UPGRADE_TO_LUCENE_10_0_0);
490+
}
484491
}

‎server/src/test/java/org/elasticsearch/index/mapper/DocumentParserContextTests.java‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -133,5 +133,6 @@ public void testCreateDynamicMapperBuilderContext() throws IOException {
133133
assertEquals(ObjectMapper.Defaults.DYNAMIC, resultFromParserContext.getDynamic());
134134
assertEquals(MapperService.MergeReason.MAPPING_UPDATE, resultFromParserContext.getMergeReason());
135135
assertFalse(resultFromParserContext.isInNestedContext());
136+
assertWarnings(SourceFieldMapper.DEPRECATION_WARNING);
136137
}
137138
}

‎server/src/test/java/org/elasticsearch/index/mapper/SourceFieldMapperTests.java‎

Lines changed: 24 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -52,7 +52,8 @@ protected void registerParameters(ParameterChecker checker) throws IOException {
5252
checker.registerConflictCheck(
5353
"enabled",
5454
topMapping(b -> b.startObject(SourceFieldMapper.NAME).field("enabled", false).endObject()),
55-
topMapping(b -> b.startObject(SourceFieldMapper.NAME).field("enabled", true).endObject())
55+
topMapping(b -> b.startObject(SourceFieldMapper.NAME).field("enabled", true).endObject()),
56+
dm -> {}
5657
);
5758
checker.registerUpdateCheck(
5859
topMapping(b -> b.startObject(SourceFieldMapper.NAME).field("enabled", true).endObject()),
@@ -62,14 +63,18 @@ protected void registerParameters(ParameterChecker checker) throws IOException {
6263
checker.registerUpdateCheck(
6364
topMapping(b -> b.startObject(SourceFieldMapper.NAME).field("mode", "stored").endObject()),
6465
topMapping(b -> b.startObject(SourceFieldMapper.NAME).field("mode", "synthetic").endObject()),
65-
dm -> assertTrue(dm.metadataMapper(SourceFieldMapper.class).isSynthetic())
66+
dm -> {
67+
assertTrue(dm.metadataMapper(SourceFieldMapper.class).isSynthetic());
68+
assertWarnings(SourceFieldMapper.DEPRECATION_WARNING);
69+
}
6670
);
6771
checker.registerConflictCheck("includes", b -> b.array("includes", "foo*"));
6872
checker.registerConflictCheck("excludes", b -> b.array("excludes", "foo*"));
6973
checker.registerConflictCheck(
7074
"mode",
7175
topMapping(b -> b.startObject(SourceFieldMapper.NAME).field("mode", "synthetic").endObject()),
72-
topMapping(b -> b.startObject(SourceFieldMapper.NAME).field("mode", "stored").endObject())
76+
topMapping(b -> b.startObject(SourceFieldMapper.NAME).field("mode", "stored").endObject()),
77+
dm -> assertWarnings(SourceFieldMapper.DEPRECATION_WARNING)
7378
);
7479
}
7580

@@ -206,20 +211,22 @@ public void testSyntheticDisabledNotSupported() {
206211
)
207212
);
208213
assertThat(e.getMessage(), containsString("Cannot set both [mode] and [enabled] parameters"));
214+
assertWarnings(SourceFieldMapper.DEPRECATION_WARNING);
209215
}
210216

211217
public void testSyntheticUpdates() throws Exception {
212218
MapperService mapperService = createMapperService("""
213219
{ "_doc" : { "_source" : { "mode" : "synthetic" } } }
214220
""");
215-
221+
assertWarnings(SourceFieldMapper.DEPRECATION_WARNING);
216222
SourceFieldMapper mapper = mapperService.documentMapper().sourceMapper();
217223
assertTrue(mapper.enabled());
218224
assertTrue(mapper.isSynthetic());
219225

220226
merge(mapperService, """
221227
{ "_doc" : { "_source" : { "mode" : "synthetic" } } }
222228
""");
229+
assertWarnings(SourceFieldMapper.DEPRECATION_WARNING);
223230
mapper = mapperService.documentMapper().sourceMapper();
224231
assertTrue(mapper.enabled());
225232
assertTrue(mapper.isSynthetic());
@@ -230,11 +237,15 @@ public void testSyntheticUpdates() throws Exception {
230237
Exception e = expectThrows(IllegalArgumentException.class, () -> merge(mapperService, """
231238
{ "_doc" : { "_source" : { "mode" : "stored" } } }
232239
"""));
240+
233241
assertThat(e.getMessage(), containsString("Cannot update parameter [mode] from [synthetic] to [stored]"));
242+
assertWarnings(SourceFieldMapper.DEPRECATION_WARNING);
234243

235244
merge(mapperService, """
236245
{ "_doc" : { "_source" : { "mode" : "disabled" } } }
237246
""");
247+
assertWarnings(SourceFieldMapper.DEPRECATION_WARNING);
248+
238249
mapper = mapperService.documentMapper().sourceMapper();
239250
assertFalse(mapper.enabled());
240251
assertFalse(mapper.isSynthetic());
@@ -270,6 +281,7 @@ public void testSupportsNonDefaultParameterValues() throws IOException {
270281
topMapping(b -> b.startObject("_source").field("mode", randomBoolean() ? "synthetic" : "stored").endObject())
271282
).documentMapper().sourceMapper();
272283
assertThat(sourceFieldMapper, notNullValue());
284+
assertWarnings(SourceFieldMapper.DEPRECATION_WARNING);
273285
}
274286
Exception e = expectThrows(
275287
MapperParsingException.class,
@@ -301,6 +313,8 @@ public void testSupportsNonDefaultParameterValues() throws IOException {
301313
.documentMapper()
302314
.sourceMapper()
303315
);
316+
assertWarnings(SourceFieldMapper.DEPRECATION_WARNING);
317+
304318
assertThat(e.getMessage(), containsString("Parameter [mode=disabled] is not allowed in source"));
305319

306320
e = expectThrows(
@@ -409,6 +423,7 @@ public void testRecoverySourceWithSyntheticSource() throws IOException {
409423
ParsedDocument doc = docMapper.parse(source(b -> { b.field("field1", "value1"); }));
410424
assertNotNull(doc.rootDoc().getField("_recovery_source"));
411425
assertThat(doc.rootDoc().getField("_recovery_source").binaryValue(), equalTo(new BytesRef("{\"field1\":\"value1\"}")));
426+
assertWarnings(SourceFieldMapper.DEPRECATION_WARNING);
412427
}
413428
{
414429
Settings settings = Settings.builder().put(INDICES_RECOVERY_SOURCE_ENABLED_SETTING.getKey(), false).build();
@@ -419,6 +434,7 @@ public void testRecoverySourceWithSyntheticSource() throws IOException {
419434
DocumentMapper docMapper = mapperService.documentMapper();
420435
ParsedDocument doc = docMapper.parse(source(b -> b.field("field1", "value1")));
421436
assertNull(doc.rootDoc().getField("_recovery_source"));
437+
assertWarnings(SourceFieldMapper.DEPRECATION_WARNING);
422438
}
423439
}
424440

@@ -613,6 +629,7 @@ public void testRecoverySourceWithLogsCustom() throws IOException {
613629
ParsedDocument doc = docMapper.parse(source(b -> { b.field("@timestamp", "2012-02-13"); }));
614630
assertNotNull(doc.rootDoc().getField("_recovery_source"));
615631
assertThat(doc.rootDoc().getField("_recovery_source").binaryValue(), equalTo(new BytesRef("{\"@timestamp\":\"2012-02-13\"}")));
632+
assertWarnings(SourceFieldMapper.DEPRECATION_WARNING);
616633
}
617634
{
618635
Settings settings = Settings.builder()
@@ -623,6 +640,7 @@ public void testRecoverySourceWithLogsCustom() throws IOException {
623640
DocumentMapper docMapper = mapperService.documentMapper();
624641
ParsedDocument doc = docMapper.parse(source(b -> b.field("@timestamp", "2012-02-13")));
625642
assertNull(doc.rootDoc().getField("_recovery_source"));
643+
assertWarnings(SourceFieldMapper.DEPRECATION_WARNING);
626644
}
627645
}
628646

@@ -691,6 +709,7 @@ public void testRecoverySourceWithTimeSeriesCustom() throws IOException {
691709
doc.rootDoc().getField("_recovery_source").binaryValue(),
692710
equalTo(new BytesRef("{\"@timestamp\":\"2012-02-13\",\"field\":\"value1\"}"))
693711
);
712+
assertWarnings(SourceFieldMapper.DEPRECATION_WARNING);
694713
}
695714
{
696715
Settings settings = Settings.builder()
@@ -704,6 +723,7 @@ public void testRecoverySourceWithTimeSeriesCustom() throws IOException {
704723
source("123", b -> b.field("@timestamp", "2012-02-13").field("field", randomAlphaOfLength(5)), null)
705724
);
706725
assertNull(doc.rootDoc().getField("_recovery_source"));
726+
assertWarnings(SourceFieldMapper.DEPRECATION_WARNING);
707727
}
708728
}
709729
}

‎server/src/test/java/org/elasticsearch/index/shard/ShardGetServiceTests.java‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@
2121
import org.elasticsearch.index.engine.VersionConflictEngineException;
2222
import org.elasticsearch.index.get.GetResult;
2323
import org.elasticsearch.index.mapper.RoutingFieldMapper;
24+
import org.elasticsearch.index.mapper.SourceFieldMapper;
2425
import org.elasticsearch.search.fetch.subphase.FetchSourceContext;
2526
import org.elasticsearch.xcontent.XContentType;
2627

@@ -114,6 +115,7 @@ public void testGetFromTranslogWithSyntheticSource() throws IOException {
114115
"mode": "synthetic"
115116
""";
116117
runGetFromTranslogWithOptions(docToIndex, sourceOptions, expectedFetchedSource, "\"long\"", 7L, true);
118+
assertWarnings(SourceFieldMapper.DEPRECATION_WARNING);
117119
}
118120

119121
public void testGetFromTranslogWithDenseVector() throws IOException {

‎test/framework/src/main/java/org/elasticsearch/index/mapper/MetadataMapperTestCase.java‎

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -38,7 +38,7 @@ protected boolean isSupportedOn(IndexVersion version) {
3838

3939
protected abstract void registerParameters(ParameterChecker checker) throws IOException;
4040

41-
private record ConflictCheck(XContentBuilder init, XContentBuilder update) {}
41+
private record ConflictCheck(XContentBuilder init, XContentBuilder update, Consumer<DocumentMapper> check) {}
4242

4343
private record UpdateCheck(XContentBuilder init, XContentBuilder update, Consumer<DocumentMapper> check) {}
4444

@@ -58,7 +58,7 @@ public void registerConflictCheck(String param, CheckedConsumer<XContentBuilder,
5858
b.startObject(fieldName());
5959
update.accept(b);
6060
b.endObject();
61-
})));
61+
}), d -> {}));
6262
}
6363

6464
/**
@@ -68,8 +68,8 @@ public void registerConflictCheck(String param, CheckedConsumer<XContentBuilder,
6868
* @param init the initial mapping
6969
* @param update the updated mapping
7070
*/
71-
public void registerConflictCheck(String param, XContentBuilder init, XContentBuilder update) {
72-
conflictChecks.put(param, new ConflictCheck(init, update));
71+
public void registerConflictCheck(String param, XContentBuilder init, XContentBuilder update, Consumer<DocumentMapper> check) {
72+
conflictChecks.put(param, new ConflictCheck(init, update, check));
7373
}
7474

7575
public void registerUpdateCheck(XContentBuilder init, XContentBuilder update, Consumer<DocumentMapper> check) {
@@ -95,6 +95,7 @@ public final void testUpdates() throws IOException {
9595
e.getMessage(),
9696
anyOf(containsString("Cannot update parameter [" + param + "]"), containsString("different [" + param + "]"))
9797
);
98+
checker.conflictChecks.get(param).check.accept(mapperService.documentMapper());
9899
}
99100
for (UpdateCheck updateCheck : checker.updateChecks) {
100101
MapperService mapperService = createMapperService(updateCheck.init);

‎test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java‎

Lines changed: 28 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -112,7 +112,6 @@
112112
import java.util.HashMap;
113113
import java.util.HashSet;
114114
import java.util.List;
115-
import java.util.Locale;
116115
import java.util.Map;
117116
import java.util.Objects;
118117
import java.util.Optional;
@@ -1835,9 +1834,10 @@ public static CreateIndexResponse createIndex(RestClient client, String name, Se
18351834

18361835
if (settings != null && settings.getAsBoolean(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), true) == false) {
18371836
expectSoftDeletesWarning(request, name);
1838-
} else if (isSyntheticSourceConfiguredInMapping(mapping)) {
1839-
request.setOptions(expectVersionSpecificWarnings(v -> v.compatible(SourceFieldMapper.DEPRECATION_WARNING)));
1840-
}
1837+
} else if (isSyntheticSourceConfiguredInMapping(mapping)
1838+
&& SourceFieldMapper.onOrAfterDeprecateModeVersion(minimumIndexVersion())) {
1839+
request.setOptions(expectVersionSpecificWarnings(v -> v.current(SourceFieldMapper.DEPRECATION_WARNING)));
1840+
}
18411841
final Response response = client.performRequest(request);
18421842
try (var parser = responseAsParser(response)) {
18431843
return TestResponseParsers.parseCreateIndexResponse(parser);
@@ -1898,8 +1898,30 @@ protected static boolean isSyntheticSourceConfiguredInMapping(String mapping) {
18981898
if (sourceMapper == null) {
18991899
return false;
19001900
}
1901-
Object mode = sourceMapper.get("mode");
1902-
return mode != null && mode.toString().toLowerCase(Locale.ROOT).equals("synthetic");
1901+
return sourceMapper.get("mode") != null;
1902+
}
1903+
1904+
@SuppressWarnings("unchecked")
1905+
protected static boolean isSyntheticSourceConfiguredInTemplate(String template) {
1906+
if (template == null) {
1907+
return false;
1908+
}
1909+
var values = XContentHelper.convertToMap(JsonXContent.jsonXContent, template, false);
1910+
for (Object value : values.values()) {
1911+
Map<String, Object> mappings = (Map<String, Object>) ((Map<String, Object>) value).get("mappings");
1912+
if (mappings == null) {
1913+
continue;
1914+
}
1915+
Map<String, Object> sourceMapper = (Map<String, Object>) mappings.get(SourceFieldMapper.NAME);
1916+
if (sourceMapper == null) {
1917+
continue;
1918+
}
1919+
Object mode = sourceMapper.get("mode");
1920+
if (mode != null) {
1921+
return true;
1922+
}
1923+
}
1924+
return false;
19031925
}
19041926

19051927
protected static Map<String, Object> getIndexSettings(String index) throws IOException {

‎x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/DeprecationChecks.java‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -96,7 +96,8 @@ private DeprecationChecks() {}
9696
IndexDeprecationChecks::checkIndexDataPath,
9797
IndexDeprecationChecks::storeTypeSettingCheck,
9898
IndexDeprecationChecks::frozenIndexSettingCheck,
99-
IndexDeprecationChecks::deprecatedCamelCasePattern
99+
IndexDeprecationChecks::deprecatedCamelCasePattern,
100+
IndexDeprecationChecks::checkSourceModeInMapping
100101
);
101102

102103
static List<BiFunction<DataStream, ClusterState, DeprecationIssue>> DATA_STREAM_CHECKS = List.of(

‎x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/IndexDeprecationChecks.java‎

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@
1616
import org.elasticsearch.index.IndexVersion;
1717
import org.elasticsearch.index.IndexVersions;
1818
import org.elasticsearch.index.engine.frozen.FrozenEngine;
19+
import org.elasticsearch.index.mapper.SourceFieldMapper;
1920
import org.elasticsearch.xpack.core.deprecation.DeprecationIssue;
2021

2122
import java.util.ArrayList;
@@ -201,6 +202,31 @@ static List<String> findInPropertiesRecursively(
201202
return issues;
202203
}
203204

205+
static DeprecationIssue checkSourceModeInMapping(IndexMetadata indexMetadata, ClusterState clusterState) {
206+
if (SourceFieldMapper.onOrAfterDeprecateModeVersion(indexMetadata.getCreationVersion())) {
207+
boolean[] useSourceMode = { false };
208+
fieldLevelMappingIssue(indexMetadata, ((mappingMetadata, sourceAsMap) -> {
209+
Object source = sourceAsMap.get("_source");
210+
if (source instanceof Map<?, ?> sourceMap) {
211+
if (sourceMap.containsKey("mode")) {
212+
useSourceMode[0] = true;
213+
}
214+
}
215+
}));
216+
if (useSourceMode[0]) {
217+
return new DeprecationIssue(
218+
DeprecationIssue.Level.CRITICAL,
219+
SourceFieldMapper.DEPRECATION_WARNING,
220+
"https://github.com/elastic/elasticsearch/pull/117172",
221+
SourceFieldMapper.DEPRECATION_WARNING,
222+
false,
223+
null
224+
);
225+
}
226+
}
227+
return null;
228+
}
229+
204230
static DeprecationIssue deprecatedCamelCasePattern(IndexMetadata indexMetadata, ClusterState clusterState) {
205231
List<String> fields = new ArrayList<>();
206232
fieldLevelMappingIssue(

0 commit comments

Comments
 (0)