ES-10063 Add multi-project support for more stats APIs

modules/ingest-common/src/internalClusterTest/java/org/elasticsearch/ingest/common/IngestRestartIT.java

@@ -20,6 +20,7 @@
 import org.elasticsearch.action.support.WriteRequest;
 import org.elasticsearch.client.internal.Requests;
 import org.elasticsearch.cluster.block.ClusterBlockException;
+import org.elasticsearch.cluster.metadata.ProjectId;
 import org.elasticsearch.cluster.node.DiscoveryNodeRole;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.concurrent.EsRejectedExecutionException;
@@ -112,7 +113,12 @@ public void testFailureInConditionalProcessor() {
         NodesStatsResponse r = clusterAdmin().prepareNodesStats(internalCluster().getNodeNames()).setIngest(true).get();
         int nodeCount = r.getNodes().size();
         for (int k = 0; k < nodeCount; k++) {
-            List<IngestStats.ProcessorStat> stats = r.getNodes().get(k).getIngestStats().processorStats().get(pipelineId);
+            List<IngestStats.ProcessorStat> stats = r.getNodes()
+                .get(k)
+                .getIngestStats()
+                .processorStats()
+                .get(ProjectId.DEFAULT)
+                .get(pipelineId);
             for (IngestStats.ProcessorStat st : stats) {
                 assertThat(st.stats().ingestCurrent(), greaterThanOrEqualTo(0L));
             }

modules/ingest-common/src/yamlRestTest/resources/rest-api-spec/test/ingest/15_info_ingest.yml

@@ -85,22 +85,22 @@ teardown:
   - gte: { ingest.total.failed: 0 }
 
   # Pipelines section
-  - is_true: ingest.pipelines.ingest_info_pipeline
-  - gte: { ingest.pipelines.ingest_info_pipeline.count: 2 }
-  - gte: { ingest.pipelines.ingest_info_pipeline.time_in_millis: 0 }
-  - match: { ingest.pipelines.ingest_info_pipeline.current: 0 }
-  - match: { ingest.pipelines.ingest_info_pipeline.failed: 0 }
-  - gt: { ingest.pipelines.ingest_info_pipeline.ingested_as_first_pipeline_in_bytes: 0 }
-  - gt: { ingest.pipelines.ingest_info_pipeline.produced_as_first_pipeline_in_bytes: 0 }
+  - is_true: "ingest.pipelines.${_project_id_prefix_}ingest_info_pipeline"
+  - gte: { "ingest.pipelines.${_project_id_prefix_}ingest_info_pipeline.count": 2 }
+  - gte: { "ingest.pipelines.${_project_id_prefix_}ingest_info_pipeline.time_in_millis": 0 }
+  - match: { "ingest.pipelines.${_project_id_prefix_}ingest_info_pipeline.current": 0 }
+  - match: { "ingest.pipelines.${_project_id_prefix_}ingest_info_pipeline.failed": 0 }
+  - gt: { "ingest.pipelines.${_project_id_prefix_}ingest_info_pipeline.ingested_as_first_pipeline_in_bytes": 0 }
+  - gt: { "ingest.pipelines.${_project_id_prefix_}ingest_info_pipeline.produced_as_first_pipeline_in_bytes": 0 }
 
   # Processors section
-  - is_true: ingest.pipelines.ingest_info_pipeline.processors.0.set
-  - match: { ingest.pipelines.ingest_info_pipeline.processors.0.set.type: "set" }
-  - is_true: ingest.pipelines.ingest_info_pipeline.processors.0.set.stats
-  - gte: { ingest.pipelines.ingest_info_pipeline.processors.0.set.stats.count: 2 }
-  - gte: { ingest.pipelines.ingest_info_pipeline.processors.0.set.stats.time_in_millis: 0 }
-  - match: { ingest.pipelines.ingest_info_pipeline.processors.0.set.stats.current: 0 }
-  - match: { ingest.pipelines.ingest_info_pipeline.processors.0.set.stats.failed: 0 }
+  - is_true: "ingest.pipelines.${_project_id_prefix_}ingest_info_pipeline.processors.0.set"
+  - match: { "ingest.pipelines.${_project_id_prefix_}ingest_info_pipeline.processors.0.set.type": "set" }
+  - is_true: "ingest.pipelines.${_project_id_prefix_}ingest_info_pipeline.processors.0.set.stats"
+  - gte: { "ingest.pipelines.${_project_id_prefix_}ingest_info_pipeline.processors.0.set.stats.count": 2 }
+  - gte: { "ingest.pipelines.${_project_id_prefix_}ingest_info_pipeline.processors.0.set.stats.time_in_millis": 0 }
+  - match: { "ingest.pipelines.${_project_id_prefix_}ingest_info_pipeline.processors.0.set.stats.current": 0 }
+  - match: { "ingest.pipelines.${_project_id_prefix_}ingest_info_pipeline.processors.0.set.stats.failed": 0 }
 
 ---
 "Test bytes_produced not increased when pipeline fails":
@@ -128,9 +128,9 @@ teardown:
   - do:
       cluster.info:
         target: [ ingest ]
-  - match: { ingest.pipelines.pipeline-1.failed: 1 }
-  - gt: { ingest.pipelines.pipeline-1.ingested_as_first_pipeline_in_bytes: 0 }
-  - match: { ingest.pipelines.pipeline-1.produced_as_first_pipeline_in_bytes: 0 }
+  - match: { "ingest.pipelines.${_project_id_prefix_}pipeline-1.failed": 1 }
+  - gt: { "ingest.pipelines.${_project_id_prefix_}pipeline-1.ingested_as_first_pipeline_in_bytes": 0 }
+  - match: { "ingest.pipelines.${_project_id_prefix_}pipeline-1.produced_as_first_pipeline_in_bytes": 0 }
 
 ---
 "Test drop processor":
@@ -156,8 +156,8 @@ teardown:
   - do:
       cluster.info:
         target: [ ingest ]
-  - gt: { ingest.pipelines.pipeline-1.ingested_as_first_pipeline_in_bytes: 0 }
-  - match: { ingest.pipelines.pipeline-1.produced_as_first_pipeline_in_bytes: 0 }
+  - gt: { "ingest.pipelines.${_project_id_prefix_}pipeline-1.ingested_as_first_pipeline_in_bytes": 0 }
+  - match: { "ingest.pipelines.${_project_id_prefix_}pipeline-1.produced_as_first_pipeline_in_bytes": 0 }
 
 ---
 "Test that pipeline processor has byte stats recorded in first pipeline":
@@ -210,11 +210,11 @@ teardown:
   - do:
       cluster.info:
         target: [ ingest ]
-  - gt: { ingest.pipelines.pipeline-1.ingested_as_first_pipeline_in_bytes: 0 }
-  - set: { ingest.pipelines.pipeline-1.ingested_as_first_pipeline_in_bytes: ingest_bytes }
-  - gt: { ingest.pipelines.pipeline-1.produced_as_first_pipeline_in_bytes: $ingest_bytes }
-  - match: { ingest.pipelines.pipeline-2.ingested_as_first_pipeline_in_bytes: 0 }
-  - match: { ingest.pipelines.pipeline-2.produced_as_first_pipeline_in_bytes: 0 }
+  - gt: { "ingest.pipelines.${_project_id_prefix_}pipeline-1.ingested_as_first_pipeline_in_bytes": 0 }
+  - set: { "ingest.pipelines.${_project_id_prefix_}pipeline-1.ingested_as_first_pipeline_in_bytes": ingest_bytes }
+  - gt: { "ingest.pipelines.${_project_id_prefix_}pipeline-1.produced_as_first_pipeline_in_bytes": $ingest_bytes }
+  - match: { "ingest.pipelines.${_project_id_prefix_}pipeline-2.ingested_as_first_pipeline_in_bytes": 0 }
+  - match: { "ingest.pipelines.${_project_id_prefix_}pipeline-2.produced_as_first_pipeline_in_bytes": 0 }
 
 ---
 "Test that final pipeline has byte stats recorded in first pipeline":
@@ -262,11 +262,11 @@ teardown:
   - do:
       cluster.info:
         target: [ ingest ]
-  - gt: { ingest.pipelines.pipeline-1.ingested_as_first_pipeline_in_bytes: 0 }
-  - set: { ingest.pipelines.pipeline-1.ingested_as_first_pipeline_in_bytes: ingest_bytes }
-  - gt: { ingest.pipelines.pipeline-1.produced_as_first_pipeline_in_bytes: $ingest_bytes }
-  - match: { ingest.pipelines.pipeline-2.ingested_as_first_pipeline_in_bytes: 0 }
-  - match: { ingest.pipelines.pipeline-2.produced_as_first_pipeline_in_bytes: 0 }
+  - gt: { "ingest.pipelines.${_project_id_prefix_}pipeline-1.ingested_as_first_pipeline_in_bytes": 0 }
+  - set: { "ingest.pipelines.${_project_id_prefix_}pipeline-1.ingested_as_first_pipeline_in_bytes": ingest_bytes }
+  - gt: { "ingest.pipelines.${_project_id_prefix_}pipeline-1.produced_as_first_pipeline_in_bytes": $ingest_bytes }
+  - match: { "ingest.pipelines.${_project_id_prefix_}pipeline-2.ingested_as_first_pipeline_in_bytes": 0 }
+  - match: { "ingest.pipelines.${_project_id_prefix_}pipeline-2.produced_as_first_pipeline_in_bytes": 0 }
 
 ---
 "Test that reroute processor has byte stats recorded in first pipeline":
@@ -327,11 +327,11 @@ teardown:
   - do:
       cluster.info:
         target: [ ingest ]
-  - gt: { ingest.pipelines.pipeline-1.ingested_as_first_pipeline_in_bytes: 0 }
-  - set: { ingest.pipelines.pipeline-1.ingested_as_first_pipeline_in_bytes: ingest_bytes }
-  - gt: { ingest.pipelines.pipeline-1.produced_as_first_pipeline_in_bytes: $ingest_bytes }
-  - match: { ingest.pipelines.pipeline-2.ingested_as_first_pipeline_in_bytes: 0 }
-  - match: { ingest.pipelines.pipeline-2.produced_as_first_pipeline_in_bytes: 0 }
+  - gt: { "ingest.pipelines.${_project_id_prefix_}pipeline-1.ingested_as_first_pipeline_in_bytes": 0 }
+  - set: { "ingest.pipelines.${_project_id_prefix_}pipeline-1.ingested_as_first_pipeline_in_bytes": ingest_bytes }
+  - gt: { "ingest.pipelines.${_project_id_prefix_}pipeline-1.produced_as_first_pipeline_in_bytes": $ingest_bytes }
+  - match: { "ingest.pipelines.${_project_id_prefix_}pipeline-2.ingested_as_first_pipeline_in_bytes": 0 }
+  - match: { "ingest.pipelines.${_project_id_prefix_}pipeline-2.produced_as_first_pipeline_in_bytes": 0 }
 
 ---
 "Test human readable byte stat fields":
@@ -360,8 +360,8 @@ teardown:
         target: [ ingest ]
         human: true
 
-  - match: { ingest.pipelines.pipeline-1.count: 1 }
-  - gt: { ingest.pipelines.pipeline-1.ingested_as_first_pipeline_in_bytes: 0 }
-  - gt: { ingest.pipelines.pipeline-1.produced_as_first_pipeline_in_bytes: 0 }
-  - is_true: ingest.pipelines.pipeline-1.ingested_as_first_pipeline
-  - is_true: ingest.pipelines.pipeline-1.produced_as_first_pipeline
+  - match: { "ingest.pipelines.${_project_id_prefix_}pipeline-1.count": 1 }
+  - gt: { "ingest.pipelines.${_project_id_prefix_}pipeline-1.ingested_as_first_pipeline_in_bytes": 0 }
+  - gt: { "ingest.pipelines.${_project_id_prefix_}pipeline-1.produced_as_first_pipeline_in_bytes": 0 }
+  - is_true: "ingest.pipelines.${_project_id_prefix_}pipeline-1.ingested_as_first_pipeline"
+  - is_true: "ingest.pipelines.${_project_id_prefix_}pipeline-1.produced_as_first_pipeline"

modules/ingest-common/src/yamlRestTest/resources/rest-api-spec/test/ingest/70_bulk.yml

@@ -88,10 +88,10 @@ teardown:
   - gte: {nodes.$master.ingest.total.failed: 0}
   - gte: {nodes.$master.ingest.total.time_in_millis: 0}
   - match: {nodes.$master.ingest.total.current: 0}
-  - gte: {nodes.$master.ingest.pipelines.pipeline1.count: 0}
-  - match: {nodes.$master.ingest.pipelines.pipeline1.failed: 0}
-  - gte: {nodes.$master.ingest.pipelines.pipeline1.time_in_millis: 0}
-  - match: {nodes.$master.ingest.pipelines.pipeline1.current: 0}
+  - gte: { "nodes.$master.ingest.pipelines.${_project_id_prefix_}pipeline1.count": 0}
+  - match: { "nodes.$master.ingest.pipelines.${_project_id_prefix_}pipeline1.failed": 0}
+  - gte: { "nodes.$master.ingest.pipelines.${_project_id_prefix_}pipeline1.time_in_millis": 0}
+  - match: { "nodes.$master.ingest.pipelines.${_project_id_prefix_}pipeline1.current": 0}
 
 ---
 "Test bulk request with default pipeline":
@@ -124,10 +124,10 @@ teardown:
   - gte: {nodes.$master.ingest.total.failed: 0}
   - gte: {nodes.$master.ingest.total.time_in_millis: 0}
   - match: {nodes.$master.ingest.total.current: 0}
-  - gte: {nodes.$master.ingest.pipelines.pipeline2.count: 0}
-  - match: {nodes.$master.ingest.pipelines.pipeline2.failed: 0}
-  - gte: {nodes.$master.ingest.pipelines.pipeline2.time_in_millis: 0}
-  - match: {nodes.$master.ingest.pipelines.pipeline2.current: 0}
+  - gte: { "nodes.$master.ingest.pipelines.${_project_id_prefix_}pipeline2.count": 0}
+  - match: { "nodes.$master.ingest.pipelines.${_project_id_prefix_}pipeline2.failed": 0}
+  - gte: { "nodes.$master.ingest.pipelines.${_project_id_prefix_}pipeline2.time_in_millis": 0}
+  - match: { "nodes.$master.ingest.pipelines.${_project_id_prefix_}pipeline2.current": 0}
 
   - do:
       get:

rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/health/40_diagnosis.yml

@@ -26,4 +26,4 @@
   - length:  { indicators.shards_availability.diagnosis: 1 }
   - is_true: indicators.shards_availability.diagnosis.0.affected_resources
   - length:  { indicators.shards_availability.diagnosis.0.affected_resources: 1 }
-  - match:   { indicators.shards_availability.diagnosis.0.affected_resources.indices.0: "red_index" }
+  - match:   { indicators.shards_availability.diagnosis.0.affected_resources.indices.0: "${_project_id_prefix_}red_index" }

rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/nodes.stats/11_indices_metrics.yml

@@ -503,9 +503,9 @@
   - gte: { nodes.$node_id.indices.mappings.total_count: 28 }
   - is_true: nodes.$node_id.indices.mappings.total_estimated_overhead
   - gte: { nodes.$node_id.indices.mappings.total_estimated_overhead_in_bytes: 26624 }
-  - match: { nodes.$node_id.indices.indices.index1.mappings.total_count: 28 }
-  - is_true: nodes.$node_id.indices.indices.index1.mappings.total_estimated_overhead
-  - match: { nodes.$node_id.indices.indices.index1.mappings.total_estimated_overhead_in_bytes: 28672 }
+  - match: { "nodes.$node_id.indices.indices.${_project_id_prefix_}index1.mappings.total_count": 28 }
+  - is_true: "nodes.$node_id.indices.indices.${_project_id_prefix_}index1.mappings.total_estimated_overhead"
+  - match: { "nodes.$node_id.indices.indices.${_project_id_prefix_}index1.mappings.total_estimated_overhead_in_bytes": 28672 }
 
 ---
 "Lucene segment level fields stats":

server/src/internalClusterTest/java/org/elasticsearch/ingest/IngestStatsNamesAndTypesIT.java

@@ -15,6 +15,7 @@
 import org.elasticsearch.action.admin.cluster.stats.ClusterStatsResponse;
 import org.elasticsearch.action.bulk.BulkRequest;
 import org.elasticsearch.action.index.IndexRequest;
+import org.elasticsearch.cluster.metadata.ProjectId;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.script.MockScriptEngine;
@@ -109,7 +110,10 @@ public void testIngestStatsNamesAndTypes() throws IOException {
             assertThat(pipelineStat.pipelineId(), equalTo("pipeline1"));
             assertThat(pipelineStat.stats().ingestCount(), equalTo(1L));
 
-            List<IngestStats.ProcessorStat> processorStats = stats.getIngestStats().processorStats().get("pipeline1");
+            List<IngestStats.ProcessorStat> processorStats = stats.getIngestStats()
+                .processorStats()
+                .get(ProjectId.DEFAULT)
+                .get("pipeline1");
             assertThat(processorStats.size(), equalTo(4));
 
             IngestStats.ProcessorStat setA = processorStats.get(0);

server/src/main/java/org/elasticsearch/TransportVersions.java

@@ -257,6 +257,7 @@ static TransportVersion def(int id) {
     public static final TransportVersion ESQL_TIME_SERIES_SOURCE_STATUS = def(9_076_0_00);
     public static final TransportVersion ESQL_HASH_OPERATOR_STATUS_OUTPUT_TIME = def(9_077_0_00);
     public static final TransportVersion ML_INFERENCE_HUGGING_FACE_CHAT_COMPLETION_ADDED = def(9_078_0_00);
+    public static final TransportVersion NODES_STATS_SUPPORTS_MULTI_PROJECT = def(9_079_0_00);
 
     /*
      * STOP! READ THIS FIRST! No, really,

server/src/main/java/org/elasticsearch/action/ActionModule.java

@@ -846,7 +846,7 @@ public void initRestHandlers(Supplier<DiscoveryNodes> nodesInCluster, Predicate<
         registerHandler.accept(new RestNodesInfoAction(settingsFilter));
         registerHandler.accept(new RestRemoteClusterInfoAction());
         registerHandler.accept(new RestNodesCapabilitiesAction());
-        registerHandler.accept(new RestNodesStatsAction());
+        registerHandler.accept(new RestNodesStatsAction(projectIdResolver));
         registerHandler.accept(new RestNodesUsageAction());
         registerHandler.accept(new RestNodesHotThreadsAction());
         registerHandler.accept(new RestClusterAllocationExplainAction());
@@ -981,7 +981,7 @@ public void initRestHandlers(Supplier<DiscoveryNodes> nodesInCluster, Predicate<
         registerHandler.accept(new RestShardsAction());
         registerHandler.accept(new RestMasterAction());
         registerHandler.accept(new RestNodesAction());
-        registerHandler.accept(new RestClusterInfoAction());
+        registerHandler.accept(new RestClusterInfoAction(projectIdResolver));
         registerHandler.accept(new RestTasksAction(nodesInCluster));
         registerHandler.accept(new RestIndicesAction(projectIdResolver));
         registerHandler.accept(new RestSegmentsAction());

server/src/main/java/org/elasticsearch/action/admin/cluster/node/stats/NodeStats.java

@@ -52,6 +52,8 @@
  */
 public class NodeStats extends BaseNodeResponse implements ChunkedToXContent {
 
+    public static final String MULTI_PROJECT_ENABLED_XCONTENT_PARAM_KEY = "multi_project_enabled_node_stats";
+
     private final long timestamp;
 
     @Nullable