Optimize loading mappings in logsdb index settings provider (elastic#120221)

Optimize mapping service usages in when determining synthetic source usage and assessing whether host.name field can be used for index sorting. This is done by excluding the part of the mapping we're not interested in.

This replaces elastic#119935 and addresses the problem differently, by just loading the parts of the mapping LogsdbIndexModeSettingsProvider is interested in and by this lowering the overhead of parsing/merging mappings that has been reported via elastic#119552.

Author: martijnvg

docs/changelog/120055.yaml

@@ -0,0 +1,5 @@
+pr: 120055
+summary: Optimize loading mappings when determining synthetic source usage and whether host.name can be sorted on.
+area: Logs
+type: enhancement
+issues: []

x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/LogsdbIndexModeSettingsProvider.java

@@ -16,6 +16,7 @@
 import org.elasticsearch.common.compress.CompressedXContent;
 import org.elasticsearch.common.regex.Regex;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.xcontent.XContentHelper;
 import org.elasticsearch.core.CheckedFunction;
 import org.elasticsearch.core.Strings;
 import org.elasticsearch.index.IndexMode;
@@ -30,12 +31,14 @@
 import org.elasticsearch.index.mapper.NumberFieldMapper;
 import org.elasticsearch.index.mapper.ObjectMapper;
 import org.elasticsearch.index.mapper.SourceFieldMapper;
+import org.elasticsearch.xcontent.XContentType;
 
 import java.io.IOException;
 import java.time.Instant;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.Locale;
+import java.util.Set;
 import java.util.function.Supplier;
 
 import static org.elasticsearch.cluster.metadata.IndexMetadata.INDEX_ROUTING_PATH;
@@ -44,6 +47,7 @@
 final class LogsdbIndexModeSettingsProvider implements IndexSettingProvider {
     private static final Logger LOGGER = LogManager.getLogger(LogsdbIndexModeSettingsProvider.class);
     private static final String LOGS_PATTERN = "logs-*-*";
+    private static final Set<String> MAPPING_INCLUDES = Set.of("_doc._source.*", "_doc.properties.host**", "_doc.subobjects");
 
     private final SyntheticSourceLicenseService syntheticSourceLicenseService;
     private final SetOnce<CheckedFunction<IndexMetadata, MapperService, IOException>> mapperServiceFactory = new SetOnce<>();
@@ -232,6 +236,19 @@ MappingHints getMappingHints(
                 // combinedTemplateMappings can be empty when creating a normal index that doesn't match any template and without mapping.
                 if (combinedTemplateMappings == null || combinedTemplateMappings.isEmpty()) {
                     combinedTemplateMappings = List.of(new CompressedXContent("{}"));
+                } else {
+                    // Filter the mapping to contain only the part this index settings provider is interested in.
+                    // This reduces the overhead of loading mappings, since mappings can be very large.
+                    // The _doc._source.mode is needed to determine synthetic source usage.
+                    // The _doc.properties.host* is needed to determine whether host.name field can be injected.
+                    // The _doc.subobjects is needed to determine whether subobjects is enabled.
+                    List<CompressedXContent> filteredMappings = new ArrayList<>(combinedTemplateMappings.size());
+                    for (CompressedXContent mappingSource : combinedTemplateMappings) {
+                        var ref = mappingSource.compressedReference();
+                        var map = XContentHelper.convertToMap(ref, true, XContentType.JSON, MAPPING_INCLUDES, Set.of()).v2();
+                        filteredMappings.add(new CompressedXContent(map));
+                    }
+                    combinedTemplateMappings = filteredMappings;
                 }
                 mapperService.merge(MapperService.SINGLE_MAPPING_NAME, combinedTemplateMappings, MapperService.MergeReason.INDEX_TEMPLATE);
                 Mapper hostName = mapperService.mappingLookup().getMapper("host.name");

x-pack/plugin/logsdb/src/yamlRestTest/resources/rest-api-spec/test/30_logsdb_default_mapping.yml

@@ -107,9 +107,116 @@ create logsdb data stream with host.name as integer and timestamp as date:
   - do:
       indices.create_data_stream:
         name: "logsdb"
+  - is_true: acknowledged
+
+  - do:
+      indices.get_data_stream:
+        name: "logsdb"
+        expand_wildcards: hidden
+  - length: { data_streams: 1 }
+  - set: { data_streams.0.indices.0.index_name: backing_index }
+
+  - do:
+      indices.get_settings:
+        index: $backing_index
+  - match: { .$backing_index.settings.index.mode: logsdb }
+  - is_false: .$backing_index.settings.index.logsdb.add_host_name_field
+  - match: { .$backing_index.settings.index.logsdb.sort_on_host_name: "true" }
+
+---
+create logsdb data stream with no host.name and timestamp as date:
+  - requires:
+      test_runner_features: [ "allowed_warnings" ]
+
+  - do:
+      cluster.put_component_template:
+        name: "logsdb-mappings"
+        body:
+          template:
+            settings:
+              mode: "logsdb"
+            mappings:
+              properties:
+                "@timestamp":
+                  type: "date"
+
+  - do:
+      indices.put_index_template:
+        name: "logsdb-index-template"
+        body:
+          index_patterns: ["logsdb"]
+          data_stream: {}
+          composed_of: ["logsdb-mappings"]
+      allowed_warnings:
+        - "index template [logsdb-index-template] has index patterns [logsdb] matching patterns from existing older templates [global] with patterns (global => [*]); this template [logsdb-index-template] will take precedence during new index creation"
 
+  - do:
+      indices.create_data_stream:
+        name: "logsdb"
   - is_true: acknowledged
 
+  - do:
+      indices.get_data_stream:
+        name: "logsdb"
+        expand_wildcards: hidden
+  - length: { data_streams: 1 }
+  - set: { data_streams.0.indices.0.index_name: backing_index }
+
+  - do:
+      indices.get_settings:
+        index: $backing_index
+  - match: { .$backing_index.settings.index.mode: logsdb }
+  - match: { .$backing_index.settings.index.logsdb.add_host_name_field: "true" }
+  - match: { .$backing_index.settings.index.logsdb.sort_on_host_name: "true" }
+
+---
+create logsdb data stream with host as keyword and timestamp as date:
+  - requires:
+      test_runner_features: [ "allowed_warnings" ]
+
+  - do:
+      cluster.put_component_template:
+        name: "logsdb-mappings"
+        body:
+          template:
+            settings:
+              mode: "logsdb"
+            mappings:
+              properties:
+                host:
+                  type: "keyword"
+                "@timestamp":
+                  type: "date"
+
+  - do:
+      indices.put_index_template:
+        name: "logsdb-index-template"
+        body:
+          index_patterns: ["logsdb"]
+          data_stream: {}
+          composed_of: ["logsdb-mappings"]
+      allowed_warnings:
+        - "index template [logsdb-index-template] has index patterns [logsdb] matching patterns from existing older templates [global] with patterns (global => [*]); this template [logsdb-index-template] will take precedence during new index creation"
+
+  - do:
+      indices.create_data_stream:
+        name: "logsdb"
+  - is_true: acknowledged
+
+  - do:
+      indices.get_data_stream:
+        name: "logsdb"
+        expand_wildcards: hidden
+  - length: { data_streams: 1 }
+  - set: { data_streams.0.indices.0.index_name: backing_index }
+
+  - do:
+      indices.get_settings:
+        index: $backing_index
+  - match: { .$backing_index.settings.index.mode: logsdb }
+  - is_false: .$backing_index.settings.index.logsdb.add_host_name_field
+  - is_false: .$backing_index.settings.index.logsdb.sort_on_host_name
+
 ---
 create logsdb data stream with host as keyword:
   - requires: