Add source mode stats to MappingStats

docs/reference/cluster/stats.asciidoc

@@ -1644,7 +1644,10 @@ The API returns the following response:
         "total_deduplicated_mapping_size": "0b",
         "total_deduplicated_mapping_size_in_bytes": 0,
         "field_types": [],
-        "runtime_field_types": []
+        "runtime_field_types": [],
+        "source_modes" : {
+          "stored": 0
+        }
       },
       "analysis": {
         "char_filter_types": [],

rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/cluster.stats/40_source_modes.yml

@@ -0,0 +1,103 @@
+---
+test source modes:
+  - requires:
+      cluster_features: ["cluster.stats.source_modes"]
+      reason: requires source modes features
+
+  - do:
+      indices.create:
+        index: test-synthetic
+        body:
+          settings:
+            index:
+              mapping:
+                source.mode: synthetic
+
+  - do:
+      indices.create:
+        index: test-stored
+
+  - do:
+      indices.create:
+        index: test-disabled
+        body:
+          settings:
+            index:
+              mapping:
+                source.mode: disabled
+
+  - do:
+      bulk:
+        refresh: true
+        body:
+          - '{ "create": { "_index": "test-synthetic" } }'
+          - '{ "name": "aaaa", "some_string": "AaAa", "some_int": 1000, "some_double": 123.456789, "some_bool": true }'
+          - '{ "create": { "_index": "test-stored" } }'
+          - '{ "name": "bbbb", "some_string": "BbBb", "some_int": 2000, "some_double": 321.987654, "some_bool": false }'
+          - '{ "create": { "_index": "test-disabled" } }'
+          - '{ "name": "cccc", "some_string": "CcCc", "some_int": 3000, "some_double": 421.484654, "some_bool": false }'
+
+  - do:
+      search:
+        index: test-*
+  - match: { hits.total.value: 3 }
+
+  - do:
+      cluster.stats: { }
+
+  - match: { indices.mappings.source_modes.disabled: 1 }
+  - match: { indices.mappings.source_modes.stored: 1 }
+  - match: { indices.mappings.source_modes.synthetic: 1 }
+
+---
+test old mapping source modes:
+  - requires:
+      cluster_features: [ "cluster.stats.source_modes" ]
+      reason: requires source modes features
+
+  - do:
+      indices.create:
+        index: test-synthetic
+        body:
+          mappings:
+            _source:
+              mode: synthetic
+
+  - do:
+      indices.create:
+        index: test-stored
+        body:
+          mappings:
+            _source:
+              mode: stored
+
+  - do:
+      indices.create:
+        index: test-disabled
+        body:
+          mappings:
+            _source:
+              mode: disabled
+
+  - do:
+      bulk:
+        refresh: true
+        body:
+          - '{ "create": { "_index": "test-synthetic" } }'
+          - '{ "name": "aaaa", "some_string": "AaAa", "some_int": 1000, "some_double": 123.456789, "some_bool": true }'
+          - '{ "create": { "_index": "test-stored" } }'
+          - '{ "name": "bbbb", "some_string": "BbBb", "some_int": 2000, "some_double": 321.987654, "some_bool": false }'
+          - '{ "create": { "_index": "test-disabled" } }'
+          - '{ "name": "cccc", "some_string": "CcCc", "some_int": 3000, "some_double": 421.484654, "some_bool": false }'
+
+  - do:
+      search:
+        index: test-*
+  - match: { hits.total.value: 3 }
+
+  - do:
+      cluster.stats: { }
+
+  - match: { indices.mappings.source_modes.disabled: 1 }
+  - match: { indices.mappings.source_modes.stored: 1 }
+  - match: { indices.mappings.source_modes.synthetic: 1 }

server/src/main/java/module-info.java

@@ -433,7 +433,8 @@
             org.elasticsearch.search.SearchFeatures,
             org.elasticsearch.script.ScriptFeatures,
             org.elasticsearch.search.retriever.RetrieversFeatures,
-            org.elasticsearch.reservedstate.service.FileSettingsFeatures;
+            org.elasticsearch.reservedstate.service.FileSettingsFeatures,
+            org.elasticsearch.action.admin.cluster.stats.ClusterStatsFeatures;
 
     uses org.elasticsearch.plugins.internal.SettingsExtension;
     uses RestExtension;

server/src/main/java/org/elasticsearch/TransportVersions.java

@@ -209,6 +209,8 @@ static TransportVersion def(int id) {
     public static final TransportVersion REINDEX_DATA_STREAMS = def(8_799_00_0);
     public static final TransportVersion ESQL_REMOVE_NODE_LEVEL_PLAN = def(8_800_00_0);
     public static final TransportVersion LOGSDB_TELEMETRY_CUSTOM_CUTOFF_DATE = def(8_801_00_0);
+    public static final TransportVersion SOURCE_MODE_TELEMETRY = def(8_802_00_0);
+
     /*
      * STOP! READ THIS FIRST! No, really,
      *        ____ _____ ___  ____  _        ____  _____    _    ____    _____ _   _ ___ ____    _____ ___ ____  ____ _____ _

server/src/main/java/org/elasticsearch/action/admin/cluster/stats/ClusterStatsFeatures.java

@@ -0,0 +1,26 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+package org.elasticsearch.action.admin.cluster.stats;
+
+import org.elasticsearch.features.FeatureSpecification;
+import org.elasticsearch.features.NodeFeature;
+
+import java.util.Set;
+
+/**
+ * Spec for cluster stats features.
+ */
+public class ClusterStatsFeatures implements FeatureSpecification {
+
+    @Override
+    public Set<NodeFeature> getFeatures() {
+        return Set.of(MappingStats.SOURCE_MODES_FEATURE);
+    }
+}

server/src/main/java/org/elasticsearch/action/admin/cluster/stats/MappingStats.java

@@ -19,6 +19,8 @@
 import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.common.unit.ByteSizeValue;
 import org.elasticsearch.core.Nullable;
+import org.elasticsearch.features.NodeFeature;
+import org.elasticsearch.index.mapper.SourceFieldMapper;
 import org.elasticsearch.xcontent.ToXContentFragment;
 import org.elasticsearch.xcontent.XContentBuilder;
 
@@ -31,6 +33,7 @@
 import java.util.HashSet;
 import java.util.IdentityHashMap;
 import java.util.List;
+import java.util.Locale;
 import java.util.Map;
 import java.util.Objects;
 import java.util.OptionalLong;
@@ -44,6 +47,8 @@
  */
 public final class MappingStats implements ToXContentFragment, Writeable {
 
+    static final NodeFeature SOURCE_MODES_FEATURE = new NodeFeature("cluster.stats.source_modes");
+
     private static final Pattern DOC_PATTERN = Pattern.compile("doc[\\[.]");
     private static final Pattern SOURCE_PATTERN = Pattern.compile("params\\._source");
 
@@ -53,6 +58,8 @@ public final class MappingStats implements ToXContentFragment, Writeable {
     public static MappingStats of(Metadata metadata, Runnable ensureNotCancelled) {
         Map<String, FieldStats> fieldTypes = new HashMap<>();
         Set<String> concreteFieldNames = new HashSet<>();
+        // Account different source modes based on index.mapping.source.mode setting:
+        Map<String, Integer> indexSourceModeUsageCount = new HashMap<>();
         Map<String, RuntimeFieldStats> runtimeFieldTypes = new HashMap<>();
         final Map<MappingMetadata, Integer> mappingCounts = new IdentityHashMap<>(metadata.getMappingsByHash().size());
         for (IndexMetadata indexMetadata : metadata) {
@@ -62,15 +69,28 @@ public static MappingStats of(Metadata metadata, Runnable ensureNotCancelled) {
                 continue;
             }
             AnalysisStats.countMapping(mappingCounts, indexMetadata);
+
+            var sourceMode = SourceFieldMapper.INDEX_MAPPER_SOURCE_MODE_SETTING.get(indexMetadata.getSettings());
+            indexSourceModeUsageCount.compute(sourceMode.toString().toLowerCase(Locale.ENGLISH), (k, v) -> v == null ? 1 : v + 1);
         }
         final AtomicLong totalFieldCount = new AtomicLong();
         final AtomicLong totalDeduplicatedFieldCount = new AtomicLong();
+        // Account different source modes based on _source.mode mapping attribute in a separate map:
+        // (Materializing the mapping into map of maps is expensive and only happens here once per unique mapping)
+        Map<String, Integer> mappingSourceModeUsageCount = new HashMap<>();
         for (Map.Entry<MappingMetadata, Integer> mappingAndCount : mappingCounts.entrySet()) {
             ensureNotCancelled.run();
             Set<String> indexFieldTypes = new HashSet<>();
             Set<String> indexRuntimeFieldTypes = new HashSet<>();
             final int count = mappingAndCount.getValue();
             final Map<String, Object> map = mappingAndCount.getKey().getSourceAsMap();
+            if (map.containsKey("_source")) {
+                Map<?, ?> sourceFieldDefinition = (Map<?, ?>) map.get("_source");
+                if (sourceFieldDefinition.containsKey("mode")) {
+                    String mode = (String) sourceFieldDefinition.get("mode");
+                    mappingSourceModeUsageCount.compute(mode.toString().toLowerCase(Locale.ENGLISH), (k, v) -> v == null ? 1 : v + 1);
+                }
+            }
             MappingVisitor.visitMapping(map, (field, fieldMapping) -> {
                 concreteFieldNames.add(field);
                 String type = null;
@@ -175,12 +195,18 @@ public static MappingStats of(Metadata metadata, Runnable ensureNotCancelled) {
         for (MappingMetadata mappingMetadata : metadata.getMappingsByHash().values()) {
             totalMappingSizeBytes += mappingMetadata.source().compressed().length;
         }
+
+        // Either pick counts based on the index setting or mapping attribute. The counts can't be combined otherwise modes get over-counted
+        // (The mappingSourceModeUsageCount is under-counted because duplicate mappings are skipped. This is at least gives some insight in
+        // case old _source.mode mapping attribute is used)
+        var sourceModeUsageCount = mappingSourceModeUsageCount.isEmpty() == false ? mappingSourceModeUsageCount : indexSourceModeUsageCount;
         return new MappingStats(
             totalFieldCount.get(),
             totalDeduplicatedFieldCount.get(),
             totalMappingSizeBytes,
             fieldTypes.values(),
-            runtimeFieldTypes.values()
+            runtimeFieldTypes.values(),
+            sourceModeUsageCount
         );
     }
 
@@ -215,17 +241,20 @@ private static int countOccurrences(String script, Pattern pattern) {
 
     private final List<FieldStats> fieldTypeStats;
     private final List<RuntimeFieldStats> runtimeFieldStats;
+    private final Map<String, Integer> sourceModeUsageCount;
 
     MappingStats(
         long totalFieldCount,
         long totalDeduplicatedFieldCount,
         long totalMappingSizeBytes,
         Collection<FieldStats> fieldTypeStats,
-        Collection<RuntimeFieldStats> runtimeFieldStats
+        Collection<RuntimeFieldStats> runtimeFieldStats,
+        Map<String, Integer> sourceModeUsageCount
     ) {
         this.totalFieldCount = totalFieldCount;
         this.totalDeduplicatedFieldCount = totalDeduplicatedFieldCount;
         this.totalMappingSizeBytes = totalMappingSizeBytes;
+        this.sourceModeUsageCount = sourceModeUsageCount;
         List<FieldStats> stats = new ArrayList<>(fieldTypeStats);
         stats.sort(Comparator.comparing(IndexFeatureStats::getName));
         this.fieldTypeStats = Collections.unmodifiableList(stats);
@@ -246,6 +275,9 @@ private static int countOccurrences(String script, Pattern pattern) {
         }
         fieldTypeStats = in.readCollectionAsImmutableList(FieldStats::new);
         runtimeFieldStats = in.readCollectionAsImmutableList(RuntimeFieldStats::new);
+        sourceModeUsageCount = in.getTransportVersion().onOrAfter(TransportVersions.SOURCE_MODE_TELEMETRY)
+            ? in.readImmutableMap(StreamInput::readString, StreamInput::readVInt)
+            : Map.of();
     }
 
     @Override
@@ -257,6 +289,9 @@ public void writeTo(StreamOutput out) throws IOException {
         }
         out.writeCollection(fieldTypeStats);
         out.writeCollection(runtimeFieldStats);
+        if (out.getTransportVersion().onOrAfter(TransportVersions.SOURCE_MODE_TELEMETRY)) {
+            out.writeMap(sourceModeUsageCount, StreamOutput::writeVInt);
+        }
     }
 
     private static OptionalLong ofNullable(Long l) {
@@ -300,6 +335,10 @@ public List<RuntimeFieldStats> getRuntimeFieldStats() {
         return runtimeFieldStats;
     }
 
+    public Map<String, Integer> getSourceModeUsageCount() {
+        return sourceModeUsageCount;
+    }
+
     @Override
     public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
         builder.startObject("mappings");
@@ -326,6 +365,12 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
             st.toXContent(builder, params);
         }
         builder.endArray();
+        builder.startObject("source_modes");
+        var entries = sourceModeUsageCount.entrySet().stream().sorted(Map.Entry.comparingByKey()).toList();
+        for (var entry : entries) {
+            builder.field(entry.getKey(), entry.getValue());
+        }
+        builder.endObject();
         builder.endObject();
         return builder;
     }
@@ -344,11 +389,19 @@ public boolean equals(Object o) {
             && Objects.equals(totalDeduplicatedFieldCount, that.totalDeduplicatedFieldCount)
             && Objects.equals(totalMappingSizeBytes, that.totalMappingSizeBytes)
             && fieldTypeStats.equals(that.fieldTypeStats)
-            && runtimeFieldStats.equals(that.runtimeFieldStats);
+            && runtimeFieldStats.equals(that.runtimeFieldStats)
+            && sourceModeUsageCount.equals(that.sourceModeUsageCount);
     }
 
     @Override
     public int hashCode() {
-        return Objects.hash(totalFieldCount, totalDeduplicatedFieldCount, totalMappingSizeBytes, fieldTypeStats, runtimeFieldStats);
+        return Objects.hash(
+            totalFieldCount,
+            totalDeduplicatedFieldCount,
+            totalMappingSizeBytes,
+            fieldTypeStats,
+            runtimeFieldStats,
+            sourceModeUsageCount
+        );
     }
 }

server/src/main/resources/META-INF/services/org.elasticsearch.features.FeatureSpecification

@@ -23,3 +23,4 @@ org.elasticsearch.search.retriever.RetrieversFeatures
 org.elasticsearch.script.ScriptFeatures
 org.elasticsearch.reservedstate.service.FileSettingsFeatures
 org.elasticsearch.cluster.routing.RoutingFeatures
+org.elasticsearch.action.admin.cluster.stats.ClusterStatsFeatures