[8.x] Add ModelRegistryMetadata to Cluster State (elastic#125150)

* Add ModelRegistryMetadata to Cluster State (elastic#121106)

This commit integrates `MinimalServiceSettings` (introduced in elastic#120560) into the cluster state for all registered models in the `ModelRegistry`.
These settings allow consumers to access configuration details without requiring asynchronous calls to retrieve full model configurations.

To ensure consistency, the cluster state metadata must remain synchronized with the models in the inference index.
If a mismatch is detected during startup, the master node performs an upgrade to load all model settings from the index.

* fix test compil

* fix serialisation

* Exclude Default Inference Endpoints from Cluster State Storage (elastic#125242)

When retrieving a default inference endpoint for the first time, the system automatically creates the endpoint.
However, unlike the `put inference model` action, the `get` action does not redirect the request to the master node.

Since elastic#121106, we rely on the assumption that every model creation (`put model`) must run on the master node, as it modifies the cluster state. However, this assumption led to a bug where the get action tries to store default inference endpoints from a different node.

This change resolves the issue by preventing default inference endpoints from being added to the cluster state. These endpoints are not strictly needed there, as they are already reported by inference services upon startup.

**Note:** This bug did not prevent the default endpoints from being used, but it caused repeated attempts to store them in the index, resulting in logging errors on every usage.

Author: jimczi

docs/changelog/121106.yaml

@@ -0,0 +1,5 @@
+pr: 121106
+summary: Add `ModelRegistryMetadata` to Cluster State
+area: Machine Learning
+type: enhancement
+issues: []

server/src/main/java/org/elasticsearch/TransportVersions.java

@@ -196,6 +196,8 @@ static TransportVersion def(int id) {
     public static final TransportVersion ML_INFERENCE_DEEPSEEK_8_19 = def(8_841_0_09);
     public static final TransportVersion ESQL_SERIALIZE_BLOCK_TYPE_CODE = def(8_841_0_10);
     public static final TransportVersion ESQL_FAILURE_FROM_REMOTE = def(8_841_0_11);
+    public static final TransportVersion ESQL_AGGREGATE_METRIC_DOUBLE_LITERAL = def(8_841_0_12);
+    public static final TransportVersion INFERENCE_MODEL_REGISTRY_METADATA_8_19 = def(8_841_0_13);
 
     /*
      * STOP! READ THIS FIRST! No, really,

server/src/main/java/org/elasticsearch/inference/MinimalServiceSettings.java

@@ -9,6 +9,12 @@
 
 package org.elasticsearch.inference;
 
+import org.elasticsearch.TransportVersion;
+import org.elasticsearch.TransportVersions;
+import org.elasticsearch.cluster.Diff;
+import org.elasticsearch.cluster.SimpleDiffable;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.core.Nullable;
 import org.elasticsearch.index.mapper.vectors.DenseVectorFieldMapper;
 import org.elasticsearch.xcontent.ConstructingObjectParser;
@@ -46,12 +52,16 @@
  * @param elementType the type of elements in the embeddings, applicable only for {@link TaskType#TEXT_EMBEDDING} (nullable).
  */
 public record MinimalServiceSettings(
+    @Nullable String service,
     TaskType taskType,
     @Nullable Integer dimensions,
     @Nullable SimilarityMeasure similarity,
     @Nullable ElementType elementType
-) implements ToXContentObject {
+) implements ServiceSettings, SimpleDiffable<MinimalServiceSettings> {
 
+    public static final String NAME = "minimal_service_settings";
+
+    public static final String SERVICE_FIELD = "service";
     public static final String TASK_TYPE_FIELD = "task_type";
     static final String DIMENSIONS_FIELD = "dimensions";
     static final String SIMILARITY_FIELD = "similarity";
@@ -61,17 +71,20 @@ public record MinimalServiceSettings(
         "model_settings",
         true,
         args -> {
-            TaskType taskType = TaskType.fromString((String) args[0]);
-            Integer dimensions = (Integer) args[1];
-            SimilarityMeasure similarity = args[2] == null ? null : SimilarityMeasure.fromString((String) args[2]);
-            DenseVectorFieldMapper.ElementType elementType = args[3] == null
+            String service = (String) args[0];
+            TaskType taskType = TaskType.fromString((String) args[1]);
+            Integer dimensions = (Integer) args[2];
+            SimilarityMeasure similarity = args[3] == null ? null : SimilarityMeasure.fromString((String) args[3]);
+            DenseVectorFieldMapper.ElementType elementType = args[4] == null
                 ? null
-                : DenseVectorFieldMapper.ElementType.fromString((String) args[3]);
-            return new MinimalServiceSettings(taskType, dimensions, similarity, elementType);
+                : DenseVectorFieldMapper.ElementType.fromString((String) args[4]);
+            return new MinimalServiceSettings(service, taskType, dimensions, similarity, elementType);
         }
     );
+    private static final String UNKNOWN_SERVICE = "_unknown_";
 
     static {
+        PARSER.declareString(ConstructingObjectParser.optionalConstructorArg(), new ParseField(SERVICE_FIELD));
         PARSER.declareString(ConstructingObjectParser.constructorArg(), new ParseField(TASK_TYPE_FIELD));
         PARSER.declareInt(ConstructingObjectParser.optionalConstructorArg(), new ParseField(DIMENSIONS_FIELD));
         PARSER.declareString(ConstructingObjectParser.optionalConstructorArg(), new ParseField(SIMILARITY_FIELD));
@@ -82,51 +95,95 @@ public static MinimalServiceSettings parse(XContentParser parser) throws IOExcep
         return PARSER.parse(parser, null);
     }
 
-    public static MinimalServiceSettings textEmbedding(int dimensions, SimilarityMeasure similarity, ElementType elementType) {
-        return new MinimalServiceSettings(TEXT_EMBEDDING, dimensions, similarity, elementType);
+    public static MinimalServiceSettings textEmbedding(
+        String serviceName,
+        int dimensions,
+        SimilarityMeasure similarity,
+        ElementType elementType
+    ) {
+        return new MinimalServiceSettings(serviceName, TEXT_EMBEDDING, dimensions, similarity, elementType);
+    }
+
+    public static MinimalServiceSettings sparseEmbedding(String serviceName) {
+        return new MinimalServiceSettings(serviceName, SPARSE_EMBEDDING, null, null, null);
     }
 
-    public static MinimalServiceSettings sparseEmbedding() {
-        return new MinimalServiceSettings(SPARSE_EMBEDDING, null, null, null);
+    public static MinimalServiceSettings rerank(String serviceName) {
+        return new MinimalServiceSettings(serviceName, RERANK, null, null, null);
     }
 
-    public static MinimalServiceSettings rerank() {
-        return new MinimalServiceSettings(RERANK, null, null, null);
+    public static MinimalServiceSettings completion(String serviceName) {
+        return new MinimalServiceSettings(serviceName, COMPLETION, null, null, null);
     }
 
-    public static MinimalServiceSettings completion() {
-        return new MinimalServiceSettings(COMPLETION, null, null, null);
+    public static MinimalServiceSettings chatCompletion(String serviceName) {
+        return new MinimalServiceSettings(serviceName, CHAT_COMPLETION, null, null, null);
     }
 
-    public static MinimalServiceSettings chatCompletion() {
-        return new MinimalServiceSettings(CHAT_COMPLETION, null, null, null);
+    public MinimalServiceSettings {
+        Objects.requireNonNull(taskType, "task type must not be null");
+        validate(taskType, dimensions, similarity, elementType);
     }
 
     public MinimalServiceSettings(Model model) {
         this(
+            model.getConfigurations().getService(),
             model.getTaskType(),
             model.getServiceSettings().dimensions(),
             model.getServiceSettings().similarity(),
             model.getServiceSettings().elementType()
         );
     }
 
-    public MinimalServiceSettings(
-        TaskType taskType,
-        @Nullable Integer dimensions,
-        @Nullable SimilarityMeasure similarity,
-        @Nullable ElementType elementType
-    ) {
-        this.taskType = Objects.requireNonNull(taskType, "task type must not be null");
-        this.dimensions = dimensions;
-        this.similarity = similarity;
-        this.elementType = elementType;
-        validate();
+    public MinimalServiceSettings(StreamInput in) throws IOException {
+        this(
+            in.readOptionalString(),
+            TaskType.fromStream(in),
+            in.readOptionalInt(),
+            in.readOptionalEnum(SimilarityMeasure.class),
+            in.readOptionalEnum(ElementType.class)
+        );
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        out.writeOptionalString(service);
+        taskType.writeTo(out);
+        out.writeOptionalInt(dimensions);
+        out.writeOptionalEnum(similarity);
+        out.writeOptionalEnum(elementType);
+    }
+
+    @Override
+    public String getWriteableName() {
+        return NAME;
+    }
+
+    @Override
+    public TransportVersion getMinimalSupportedVersion() {
+        return TransportVersions.INFERENCE_MODEL_REGISTRY_METADATA_8_19;
+    }
+
+    @Override
+    public ToXContentObject getFilteredXContentObject() {
+        return this::toXContent;
+    }
+
+    @Override
+    public String modelId() {
+        return null;
+    }
+
+    public static Diff<MinimalServiceSettings> readDiffFrom(StreamInput in) throws IOException {
+        return SimpleDiffable.readDiffFrom(MinimalServiceSettings::new, in);
     }
 
     @Override
     public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
         builder.startObject();
+        if (service != null) {
+            builder.field(SERVICE_FIELD, service);
+        }
         builder.field(TASK_TYPE_FIELD, taskType.toString());
         if (dimensions != null) {
             builder.field(DIMENSIONS_FIELD, dimensions);
@@ -143,7 +200,8 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
     @Override
     public String toString() {
         final StringBuilder sb = new StringBuilder();
-        sb.append("task_type=").append(taskType);
+        sb.append("service=").append(service);
+        sb.append(", task_type=").append(taskType);
         if (dimensions != null) {
             sb.append(", dimensions=").append(dimensions);
         }
@@ -156,31 +214,46 @@ public String toString() {
         return sb.toString();
     }
 
-    private void validate() {
+    private static void validate(TaskType taskType, Integer dimensions, SimilarityMeasure similarity, ElementType elementType) {
         switch (taskType) {
             case TEXT_EMBEDDING:
-                validateFieldPresent(DIMENSIONS_FIELD, dimensions);
-                validateFieldPresent(SIMILARITY_FIELD, similarity);
-                validateFieldPresent(ELEMENT_TYPE_FIELD, elementType);
+                validateFieldPresent(DIMENSIONS_FIELD, dimensions, taskType);
+                validateFieldPresent(SIMILARITY_FIELD, similarity, taskType);
+                validateFieldPresent(ELEMENT_TYPE_FIELD, elementType, taskType);
                 break;
 
             default:
-                validateFieldNotPresent(DIMENSIONS_FIELD, dimensions);
-                validateFieldNotPresent(SIMILARITY_FIELD, similarity);
-                validateFieldNotPresent(ELEMENT_TYPE_FIELD, elementType);
+                validateFieldNotPresent(DIMENSIONS_FIELD, dimensions, taskType);
+                validateFieldNotPresent(SIMILARITY_FIELD, similarity, taskType);
+                validateFieldNotPresent(ELEMENT_TYPE_FIELD, elementType, taskType);
                 break;
         }
     }
 
-    private void validateFieldPresent(String field, Object fieldValue) {
+    private static void validateFieldPresent(String field, Object fieldValue, TaskType taskType) {
         if (fieldValue == null) {
             throw new IllegalArgumentException("required [" + field + "] field is missing for task_type [" + taskType.name() + "]");
         }
     }
 
-    private void validateFieldNotPresent(String field, Object fieldValue) {
+    private static void validateFieldNotPresent(String field, Object fieldValue, TaskType taskType) {
         if (fieldValue != null) {
             throw new IllegalArgumentException("[" + field + "] is not allowed for task_type [" + taskType.name() + "]");
         }
     }
+
+    public ModelConfigurations toModelConfigurations(String inferenceEntityId) {
+        return new ModelConfigurations(inferenceEntityId, taskType, service == null ? UNKNOWN_SERVICE : service, this);
+    }
+
+    /**
+     * Checks if the given {@link MinimalServiceSettings} is equivalent to the current definition.
+     */
+    public boolean canMergeWith(MinimalServiceSettings other) {
+        return taskType == other.taskType
+            && Objects.equals(dimensions, other.dimensions)
+            && similarity == other.similarity
+            && elementType == other.elementType
+            && (service == null || service.equals(other.service));
+    }
 }

server/src/test/java/org/elasticsearch/inference/MinimalServiceSettingsTests.java

@@ -16,8 +16,7 @@
 import java.io.IOException;
 
 public class MinimalServiceSettingsTests extends AbstractXContentTestCase<MinimalServiceSettings> {
-    @Override
-    protected MinimalServiceSettings createTestInstance() {
+    public static MinimalServiceSettings randomInstance() {
         TaskType taskType = randomFrom(TaskType.values());
         Integer dimensions = null;
         SimilarityMeasure similarity = null;
@@ -28,7 +27,12 @@ protected MinimalServiceSettings createTestInstance() {
             similarity = randomFrom(SimilarityMeasure.values());
             elementType = randomFrom(DenseVectorFieldMapper.ElementType.values());
         }
-        return new MinimalServiceSettings(taskType, dimensions, similarity, elementType);
+        return new MinimalServiceSettings(randomBoolean() ? null : randomAlphaOfLength(10), taskType, dimensions, similarity, elementType);
+    }
+
+    @Override
+    protected MinimalServiceSettings createTestInstance() {
+        return randomInstance();
     }
 
     @Override

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/action/GetInferenceModelAction.java

@@ -45,10 +45,7 @@ public static class Request extends AcknowledgedRequest<GetInferenceModelAction.
         private final boolean persistDefaultConfig;
 
         public Request(String inferenceEntityId, TaskType taskType) {
-            super(TRAPPY_IMPLICIT_DEFAULT_MASTER_NODE_TIMEOUT, DEFAULT_ACK_TIMEOUT);
-            this.inferenceEntityId = Objects.requireNonNull(inferenceEntityId);
-            this.taskType = Objects.requireNonNull(taskType);
-            this.persistDefaultConfig = PERSIST_DEFAULT_CONFIGS;
+            this(inferenceEntityId, taskType, PERSIST_DEFAULT_CONFIGS);
         }
 
         public Request(String inferenceEntityId, TaskType taskType, boolean persistDefaultConfig) {
@@ -68,7 +65,6 @@ public Request(StreamInput in) throws IOException {
             } else {
                 this.persistDefaultConfig = PERSIST_DEFAULT_CONFIGS;
             }
-
         }
 
         public String getInferenceEntityId() {

x-pack/plugin/esql/qa/server/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/multi_node/SemanticMatchIT.java

@@ -17,7 +17,9 @@
 @ThreadLeakFilters(filters = TestClustersThreadFilter.class)
 public class SemanticMatchIT extends SemanticMatchTestCase {
     @ClassRule
-    public static ElasticsearchCluster cluster = Clusters.testCluster(spec -> spec.plugin("inference-service-test"));
+    public static ElasticsearchCluster cluster = Clusters.testCluster(
+        spec -> spec.module("x-pack-inference").plugin("inference-service-test")
+    );
 
     @Override
     protected String getTestRestCluster() {

x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/SemanticMatchTestCase.java

@@ -18,6 +18,7 @@
 import java.io.IOException;
 import java.util.Map;
 
+import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.core.StringContains.containsString;
 
 public abstract class SemanticMatchTestCase extends ESRestTestCase {
@@ -88,16 +89,22 @@ public void setUpTextEmbeddingInferenceEndpoint() throws IOException {
         Request request = new Request("PUT", "_inference/text_embedding/test_dense_inference");
         request.setJsonEntity("""
                   {
-                   "service": "test_service",
+                   "service": "text_embedding_test_service",
                    "service_settings": {
                      "model": "my_model",
-                     "api_key": "abc64"
+                     "api_key": "abc64",
+                     "dimensions": 128
                    },
                    "task_settings": {
                    }
                  }
             """);
-        adminClient().performRequest(request);
+        try {
+            adminClient().performRequest(request);
+        } catch (ResponseException exc) {
+            // in case the removal failed
+            assertThat(exc.getResponse().getStatusLine().getStatusCode(), equalTo(400));
+        }
     }
 
     @After

x-pack/plugin/inference/qa/inference-service-tests/src/javaRestTest/java/org/elasticsearch/xpack/inference/DefaultEndPointsIT.java

@@ -24,6 +24,7 @@
 import java.util.concurrent.CountDownLatch;
 
 import static org.hamcrest.Matchers.empty;
+import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.hasSize;
 import static org.hamcrest.Matchers.is;
 import static org.hamcrest.Matchers.oneOf;
@@ -54,6 +55,25 @@ public void testGet() throws IOException {
         assertDefaultRerankConfig(rerankModel);
     }
 
+    public void testDefaultModels() throws IOException {
+        var elserModel = getModel(ElasticsearchInternalService.DEFAULT_ELSER_ID);
+        assertDefaultElserConfig(elserModel);
+
+        var e5Model = getModel(ElasticsearchInternalService.DEFAULT_E5_ID);
+        assertDefaultE5Config(e5Model);
+
+        var rerankModel = getModel(ElasticsearchInternalService.DEFAULT_RERANK_ID);
+        assertDefaultRerankConfig(rerankModel);
+
+        putModel("my-model", mockCompletionServiceModelConfig(TaskType.SPARSE_EMBEDDING));
+        var registeredModels = getMinimalConfigs();
+        assertThat(registeredModels.size(), equalTo(1));
+        assertTrue(registeredModels.containsKey("my-model"));
+        assertFalse(registeredModels.containsKey(ElasticsearchInternalService.DEFAULT_E5_ID));
+        assertFalse(registeredModels.containsKey(ElasticsearchInternalService.DEFAULT_ELSER_ID));
+        assertFalse(registeredModels.containsKey(ElasticsearchInternalService.DEFAULT_RERANK_ID));
+    }
+
     @SuppressWarnings("unchecked")
     public void testInferDeploysDefaultElser() throws IOException {
         var model = getModel(ElasticsearchInternalService.DEFAULT_ELSER_ID);