Skip to content

Commit 87987e7

Browse files
salvatore-campagnaelasticmachine
salvatore-campagna
and
elasticmachine
authored
[8.x] Allow synthetic source and disabled source for standard indices (#114817) (#114866)
* Allow synthetic source and disabled source for standard indices (#114817) When using the index.mapping.source.mode setting we need to make sure that it takes precedence and that is used also when standard index mode is used. Without this patch we always return stored source if _source.mode is not used and the setting is. Relates #114433 (cherry picked from commit 3af4d67) # Conflicts: # server/src/main/java/org/elasticsearch/index/mapper/SourceFieldMapper.java * fix: conflict resolution mistake * fix: error message --------- Co-authored-by: Elastic Machine <[email protected]>
1 parent a25c8f2 commit 87987e7

File tree

3 files changed

+240
-24
lines changed

3 files changed

+240
-24
lines changed

‎server/src/main/java/org/elasticsearch/index/mapper/SourceFieldMapper.java‎

Lines changed: 77 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -92,6 +92,42 @@ public enum Mode {
9292
true
9393
);
9494

95+
private static final SourceFieldMapper DEFAULT_DISABLED = new SourceFieldMapper(
96+
Mode.DISABLED,
97+
Explicit.IMPLICIT_TRUE,
98+
Strings.EMPTY_ARRAY,
99+
Strings.EMPTY_ARRAY,
100+
null,
101+
true
102+
);
103+
104+
private static final SourceFieldMapper DEFAULT_DISABLED_NO_RECOVERY_SOURCE = new SourceFieldMapper(
105+
Mode.DISABLED,
106+
Explicit.IMPLICIT_TRUE,
107+
Strings.EMPTY_ARRAY,
108+
Strings.EMPTY_ARRAY,
109+
null,
110+
false
111+
);
112+
113+
private static final SourceFieldMapper DEFAULT_SYNTHETIC = new SourceFieldMapper(
114+
Mode.SYNTHETIC,
115+
Explicit.IMPLICIT_TRUE,
116+
Strings.EMPTY_ARRAY,
117+
Strings.EMPTY_ARRAY,
118+
null,
119+
true
120+
);
121+
122+
private static final SourceFieldMapper DEFAULT_SYNTHETIC_NO_RECOVERY_SOURCE = new SourceFieldMapper(
123+
Mode.SYNTHETIC,
124+
Explicit.IMPLICIT_TRUE,
125+
Strings.EMPTY_ARRAY,
126+
Strings.EMPTY_ARRAY,
127+
null,
128+
false
129+
);
130+
95131
private static final SourceFieldMapper DEFAULT_NO_RECOVERY_SOURCE = new SourceFieldMapper(
96132
null,
97133
Explicit.IMPLICIT_TRUE,
@@ -301,7 +337,7 @@ public SourceFieldMapper build() {
301337
? INDEX_MAPPER_SOURCE_MODE_SETTING.get(settings)
302338
: mode.get();
303339
if (isDefault(sourceMode)) {
304-
return resolveSourceMode(indexMode, sourceMode, enableRecoverySource);
340+
return resolveSourceMode(indexMode, sourceMode == null ? Mode.STORED : sourceMode, enableRecoverySource);
305341

306342
}
307343
if (supportsNonDefaultParameterValues == false) {
@@ -344,27 +380,38 @@ public SourceFieldMapper build() {
344380
}
345381

346382
private static SourceFieldMapper resolveSourceMode(final IndexMode indexMode, final Mode sourceMode, boolean enableRecoverySource) {
347-
if (indexMode == IndexMode.STANDARD) {
348-
return enableRecoverySource ? DEFAULT : DEFAULT_NO_RECOVERY_SOURCE;
349-
}
350-
final SourceFieldMapper syntheticWithoutRecoverySource = indexMode == IndexMode.TIME_SERIES
351-
? TSDB_DEFAULT_NO_RECOVERY_SOURCE
352-
: LOGSDB_DEFAULT_NO_RECOVERY_SOURCE;
353-
final SourceFieldMapper syntheticWithRecoverySource = indexMode == IndexMode.TIME_SERIES ? TSDB_DEFAULT : LOGSDB_DEFAULT;
354-
final SourceFieldMapper storedWithoutRecoverySource = indexMode == IndexMode.TIME_SERIES
355-
? TSDB_DEFAULT_NO_RECOVERY_SOURCE_STORED
356-
: LOGSDB_DEFAULT_NO_RECOVERY_SOURCE_STORED;
357-
final SourceFieldMapper storedWithRecoverySource = indexMode == IndexMode.TIME_SERIES ? TSDB_DEFAULT_STORED : LOGSDB_DEFAULT_STORED;
358-
359-
switch (sourceMode) {
360-
case SYNTHETIC:
361-
return enableRecoverySource ? syntheticWithRecoverySource : syntheticWithoutRecoverySource;
362-
case STORED:
363-
return enableRecoverySource ? storedWithRecoverySource : storedWithoutRecoverySource;
364-
case DISABLED:
365-
throw new IllegalArgumentException("_source cannot be disabled in index using [" + indexMode + "] index mode");
383+
switch (indexMode) {
384+
case STANDARD:
385+
switch (sourceMode) {
386+
case SYNTHETIC:
387+
return enableRecoverySource ? DEFAULT_SYNTHETIC : DEFAULT_SYNTHETIC_NO_RECOVERY_SOURCE;
388+
case STORED:
389+
return enableRecoverySource ? DEFAULT : DEFAULT_NO_RECOVERY_SOURCE;
390+
case DISABLED:
391+
return enableRecoverySource ? DEFAULT_DISABLED : DEFAULT_DISABLED_NO_RECOVERY_SOURCE;
392+
default:
393+
throw new IllegalArgumentException("Unsupported source mode: " + sourceMode);
394+
}
395+
case TIME_SERIES:
396+
case LOGSDB:
397+
switch (sourceMode) {
398+
case SYNTHETIC:
399+
return enableRecoverySource
400+
? (indexMode == IndexMode.TIME_SERIES ? TSDB_DEFAULT : LOGSDB_DEFAULT)
401+
: (indexMode == IndexMode.TIME_SERIES ? TSDB_DEFAULT_NO_RECOVERY_SOURCE : LOGSDB_DEFAULT_NO_RECOVERY_SOURCE);
402+
case STORED:
403+
return enableRecoverySource
404+
? (indexMode == IndexMode.TIME_SERIES ? TSDB_DEFAULT_STORED : LOGSDB_DEFAULT_STORED)
405+
: (indexMode == IndexMode.TIME_SERIES
406+
? TSDB_DEFAULT_NO_RECOVERY_SOURCE_STORED
407+
: LOGSDB_DEFAULT_NO_RECOVERY_SOURCE_STORED);
408+
case DISABLED:
409+
throw new IllegalArgumentException("_source can not be disabled in index using [" + indexMode + "] index mode");
410+
default:
411+
throw new IllegalArgumentException("Unsupported source mode: " + sourceMode);
412+
}
366413
default:
367-
throw new IllegalStateException("Unexpected value: " + sourceMode);
414+
throw new IllegalArgumentException("Unsupported index mode: " + indexMode);
368415
}
369416
}
370417

@@ -378,7 +425,7 @@ private static SourceFieldMapper resolveSourceMode(final IndexMode indexMode, fi
378425
return enableRecoverySource ? TSDB_LEGACY_DEFAULT : TSDB_LEGACY_DEFAULT_NO_RECOVERY_SOURCE;
379426
}
380427
}
381-
return resolveSourceMode(indexMode, settingSourceMode, enableRecoverySource);
428+
return resolveSourceMode(indexMode, settingSourceMode == null ? Mode.STORED : settingSourceMode, enableRecoverySource);
382429
},
383430
c -> new Builder(
384431
c.getIndexSettings().getMode(),
@@ -548,4 +595,12 @@ public SourceLoader newSourceLoader(Mapping mapping, SourceFieldMetrics metrics)
548595
public boolean isSynthetic() {
549596
return mode == Mode.SYNTHETIC;
550597
}
598+
599+
public boolean isDisabled() {
600+
return mode == Mode.DISABLED;
601+
}
602+
603+
public boolean isStored() {
604+
return mode == null || mode == Mode.STORED;
605+
}
551606
}

‎server/src/test/java/org/elasticsearch/index/mapper/SourceFieldMapperTests.java‎

Lines changed: 161 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -443,6 +443,167 @@ public void testRecoverySourceWithLogs() throws IOException {
443443
}
444444
}
445445

446+
public void testStandardIndexModeWithSourceModeSetting() throws IOException {
447+
// Test for IndexMode.STANDARD
448+
{
449+
final XContentBuilder mappings = topMapping(b -> {});
450+
final Settings settings = Settings.builder()
451+
.put(IndexSettings.MODE.getKey(), IndexMode.STANDARD.name())
452+
.put(SourceFieldMapper.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), SourceFieldMapper.Mode.SYNTHETIC)
453+
.build();
454+
final MapperService mapperService = createMapperService(settings, mappings);
455+
DocumentMapper docMapper = mapperService.documentMapper();
456+
assertTrue(docMapper.sourceMapper().isSynthetic());
457+
}
458+
{
459+
final XContentBuilder mappings = topMapping(b -> {});
460+
final Settings settings = Settings.builder()
461+
.put(IndexSettings.MODE.getKey(), IndexMode.STANDARD.name())
462+
.put(SourceFieldMapper.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), SourceFieldMapper.Mode.STORED)
463+
.build();
464+
final MapperService mapperService = createMapperService(settings, mappings);
465+
final DocumentMapper docMapper = mapperService.documentMapper();
466+
assertTrue(docMapper.sourceMapper().isStored());
467+
}
468+
{
469+
final XContentBuilder mappings = topMapping(b -> {});
470+
final Settings settings = Settings.builder()
471+
.put(IndexSettings.MODE.getKey(), IndexMode.STANDARD.name())
472+
.put(SourceFieldMapper.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), SourceFieldMapper.Mode.DISABLED)
473+
.build();
474+
final MapperService mapperService = createMapperService(settings, mappings);
475+
final DocumentMapper docMapper = mapperService.documentMapper();
476+
assertTrue(docMapper.sourceMapper().isDisabled());
477+
}
478+
479+
// Test for IndexMode.LOGSDB
480+
{
481+
final XContentBuilder mappings = topMapping(b -> {});
482+
final Settings settings = Settings.builder()
483+
.put(IndexSettings.MODE.getKey(), IndexMode.LOGSDB.name())
484+
.put(SourceFieldMapper.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), SourceFieldMapper.Mode.SYNTHETIC)
485+
.build();
486+
final MapperService mapperService = createMapperService(settings, mappings);
487+
DocumentMapper docMapper = mapperService.documentMapper();
488+
assertTrue(docMapper.sourceMapper().isSynthetic());
489+
}
490+
{
491+
final XContentBuilder mappings = topMapping(b -> {});
492+
final Settings settings = Settings.builder()
493+
.put(IndexSettings.MODE.getKey(), IndexMode.LOGSDB.name())
494+
.put(SourceFieldMapper.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), SourceFieldMapper.Mode.STORED)
495+
.build();
496+
final MapperService mapperService = createMapperService(settings, mappings);
497+
final DocumentMapper docMapper = mapperService.documentMapper();
498+
assertTrue(docMapper.sourceMapper().isStored());
499+
}
500+
{
501+
final XContentBuilder mappings = topMapping(b -> {});
502+
final Settings settings = Settings.builder()
503+
.put(IndexSettings.MODE.getKey(), IndexMode.LOGSDB.name())
504+
.put(SourceFieldMapper.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), SourceFieldMapper.Mode.DISABLED)
505+
.build();
506+
var ex = expectThrows(MapperParsingException.class, () -> createMapperService(settings, mappings));
507+
assertEquals("Failed to parse mapping: _source can not be disabled in index using [logsdb] index mode", ex.getMessage());
508+
}
509+
510+
// Test for IndexMode.TIME_SERIES
511+
{
512+
final String mappings = """
513+
{
514+
"_doc" : {
515+
"properties": {
516+
"routing_field": {
517+
"type": "keyword",
518+
"time_series_dimension": true
519+
}
520+
}
521+
}
522+
}
523+
""";
524+
final Settings settings = Settings.builder()
525+
.put(IndexSettings.MODE.getKey(), IndexMode.TIME_SERIES.name())
526+
.put(SourceFieldMapper.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), SourceFieldMapper.Mode.SYNTHETIC)
527+
.put(IndexMetadata.INDEX_ROUTING_PATH.getKey(), "routing_field")
528+
.build();
529+
final MapperService mapperService = createMapperService(settings, mappings);
530+
DocumentMapper docMapper = mapperService.documentMapper();
531+
assertTrue(docMapper.sourceMapper().isSynthetic());
532+
}
533+
{
534+
final String mappings = """
535+
{
536+
"_doc" : {
537+
"properties": {
538+
"routing_field": {
539+
"type": "keyword",
540+
"time_series_dimension": true
541+
}
542+
}
543+
}
544+
}
545+
""";
546+
final Settings settings = Settings.builder()
547+
.put(IndexSettings.MODE.getKey(), IndexMode.TIME_SERIES.name())
548+
.put(SourceFieldMapper.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), SourceFieldMapper.Mode.STORED)
549+
.put(IndexMetadata.INDEX_ROUTING_PATH.getKey(), "routing_field")
550+
.build();
551+
final MapperService mapperService = createMapperService(settings, mappings);
552+
final DocumentMapper docMapper = mapperService.documentMapper();
553+
assertTrue(docMapper.sourceMapper().isStored());
554+
}
555+
{
556+
final String mappings = """
557+
{
558+
"_doc" : {
559+
"properties": {
560+
"routing_field": {
561+
"type": "keyword",
562+
"time_series_dimension": true
563+
}
564+
}
565+
}
566+
}
567+
""";
568+
final Settings settings = Settings.builder()
569+
.put(IndexSettings.MODE.getKey(), IndexMode.TIME_SERIES.name())
570+
.put(SourceFieldMapper.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), SourceFieldMapper.Mode.DISABLED)
571+
.put(IndexMetadata.INDEX_ROUTING_PATH.getKey(), "routing_field")
572+
.build();
573+
var ex = expectThrows(MapperParsingException.class, () -> createMapperService(settings, mappings));
574+
assertEquals("Failed to parse mapping: _source can not be disabled in index using [time_series] index mode", ex.getMessage());
575+
}
576+
577+
// Test cases without IndexMode (default to standard)
578+
{
579+
final XContentBuilder mappings = topMapping(b -> {});
580+
final Settings settings = Settings.builder()
581+
.put(SourceFieldMapper.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), SourceFieldMapper.Mode.SYNTHETIC)
582+
.build();
583+
final MapperService mapperService = createMapperService(settings, mappings);
584+
DocumentMapper docMapper = mapperService.documentMapper();
585+
assertTrue(docMapper.sourceMapper().isSynthetic());
586+
}
587+
{
588+
final XContentBuilder mappings = topMapping(b -> {});
589+
final Settings settings = Settings.builder()
590+
.put(SourceFieldMapper.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), SourceFieldMapper.Mode.STORED)
591+
.build();
592+
final MapperService mapperService = createMapperService(settings, mappings);
593+
final DocumentMapper docMapper = mapperService.documentMapper();
594+
assertTrue(docMapper.sourceMapper().isStored());
595+
}
596+
{
597+
final XContentBuilder mappings = topMapping(b -> {});
598+
final Settings settings = Settings.builder()
599+
.put(SourceFieldMapper.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), SourceFieldMapper.Mode.DISABLED)
600+
.build();
601+
final MapperService mapperService = createMapperService(settings, mappings);
602+
final DocumentMapper docMapper = mapperService.documentMapper();
603+
assertTrue(docMapper.sourceMapper().isDisabled());
604+
}
605+
}
606+
446607
public void testRecoverySourceWithLogsCustom() throws IOException {
447608
XContentBuilder mappings = topMapping(b -> b.startObject(SourceFieldMapper.NAME).field("mode", "synthetic").endObject());
448609
{

‎x-pack/plugin/logsdb/src/yamlRestTest/resources/rest-api-spec/test/40_source_mode_setting.yml‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -559,7 +559,7 @@ create an index with time_series index mode and disabled source:
559559
time_series_dimension: true
560560

561561
- match: { error.type: "mapper_parsing_exception" }
562-
- match: { error.reason: "Failed to parse mapping: _source cannot be disabled in index using [time_series] index mode" }
562+
- match: { error.reason: "Failed to parse mapping: _source can not be disabled in index using [time_series] index mode" }
563563

564564
---
565565
create an index with logsdb index mode and disabled source:
@@ -574,7 +574,7 @@ create an index with logsdb index mode and disabled source:
574574
mapping.source.mode: disabled
575575

576576
- match: { error.type: "mapper_parsing_exception" }
577-
- match: { error.reason: "Failed to parse mapping: _source cannot be disabled in index using [logsdb] index mode" }
577+
- match: { error.reason: "Failed to parse mapping: _source can not be disabled in index using [logsdb] index mode" }
578578

579579
---
580580
modify final setting after index creation:

0 commit comments

Comments
 (0)