Add ModelRegistryMetadata to Cluster State (elastic#121106)

This commit integrates `MinimalServiceSettings` (introduced in elastic#120560) into the cluster state for all registered models in the `ModelRegistry`.
These settings allow consumers to access configuration details without requiring asynchronous calls to retrieve full model configurations.

To ensure consistency, the cluster state metadata must remain synchronized with the models in the inference index.
If a mismatch is detected during startup, the master node performs an upgrade to load all model settings from the index.

Author: jimczi

docs/changelog/121106.yaml

@@ -0,0 +1,5 @@
+pr: 121106
+summary: Add `ModelRegistryMetadata` to Cluster State
+area: Machine Learning
+type: enhancement
+issues: []

server/src/main/java/org/elasticsearch/TransportVersions.java

@@ -187,6 +187,7 @@ static TransportVersion def(int id) {
     public static final TransportVersion ML_INFERENCE_DEEPSEEK = def(9_029_00_0);
     public static final TransportVersion ESQL_FAILURE_FROM_REMOTE = def(9_030_00_0);
     public static final TransportVersion INDEX_RESHARDING_METADATA = def(9_031_0_00);
+    public static final TransportVersion INFERENCE_MODEL_REGISTRY_METADATA = def(9_032_0_00);
 
     /*
      * STOP! READ THIS FIRST! No, really,

server/src/main/java/org/elasticsearch/inference/MinimalServiceSettings.java

@@ -9,6 +9,12 @@
 
 package org.elasticsearch.inference;
 
+import org.elasticsearch.TransportVersion;
+import org.elasticsearch.TransportVersions;
+import org.elasticsearch.cluster.Diff;
+import org.elasticsearch.cluster.SimpleDiffable;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.core.Nullable;
 import org.elasticsearch.index.mapper.vectors.DenseVectorFieldMapper;
 import org.elasticsearch.xcontent.ConstructingObjectParser;
@@ -46,12 +52,16 @@
  * @param elementType the type of elements in the embeddings, applicable only for {@link TaskType#TEXT_EMBEDDING} (nullable).
  */
 public record MinimalServiceSettings(
+    @Nullable String service,
     TaskType taskType,
     @Nullable Integer dimensions,
     @Nullable SimilarityMeasure similarity,
     @Nullable ElementType elementType
-) implements ToXContentObject {
+) implements ServiceSettings, SimpleDiffable<MinimalServiceSettings> {
 
+    public static final String NAME = "minimal_service_settings";
+
+    public static final String SERVICE_FIELD = "service";
     public static final String TASK_TYPE_FIELD = "task_type";
     static final String DIMENSIONS_FIELD = "dimensions";
     static final String SIMILARITY_FIELD = "similarity";
@@ -61,17 +71,20 @@ public record MinimalServiceSettings(
         "model_settings",
         true,
         args -> {
-            TaskType taskType = TaskType.fromString((String) args[0]);
-            Integer dimensions = (Integer) args[1];
-            SimilarityMeasure similarity = args[2] == null ? null : SimilarityMeasure.fromString((String) args[2]);
-            DenseVectorFieldMapper.ElementType elementType = args[3] == null
+            String service = (String) args[0];
+            TaskType taskType = TaskType.fromString((String) args[1]);
+            Integer dimensions = (Integer) args[2];
+            SimilarityMeasure similarity = args[3] == null ? null : SimilarityMeasure.fromString((String) args[3]);
+            DenseVectorFieldMapper.ElementType elementType = args[4] == null
                 ? null
-                : DenseVectorFieldMapper.ElementType.fromString((String) args[3]);
-            return new MinimalServiceSettings(taskType, dimensions, similarity, elementType);
+                : DenseVectorFieldMapper.ElementType.fromString((String) args[4]);
+            return new MinimalServiceSettings(service, taskType, dimensions, similarity, elementType);
         }
     );
+    private static final String UNKNOWN_SERVICE = "_unknown_";
 
     static {
+        PARSER.declareString(ConstructingObjectParser.optionalConstructorArg(), new ParseField(SERVICE_FIELD));
         PARSER.declareString(ConstructingObjectParser.constructorArg(), new ParseField(TASK_TYPE_FIELD));
         PARSER.declareInt(ConstructingObjectParser.optionalConstructorArg(), new ParseField(DIMENSIONS_FIELD));
         PARSER.declareString(ConstructingObjectParser.optionalConstructorArg(), new ParseField(SIMILARITY_FIELD));
@@ -82,51 +95,95 @@ public static MinimalServiceSettings parse(XContentParser parser) throws IOExcep
         return PARSER.parse(parser, null);
     }
 
-    public static MinimalServiceSettings textEmbedding(int dimensions, SimilarityMeasure similarity, ElementType elementType) {
-        return new MinimalServiceSettings(TEXT_EMBEDDING, dimensions, similarity, elementType);
+    public static MinimalServiceSettings textEmbedding(
+        String serviceName,
+        int dimensions,
+        SimilarityMeasure similarity,
+        ElementType elementType
+    ) {
+        return new MinimalServiceSettings(serviceName, TEXT_EMBEDDING, dimensions, similarity, elementType);
+    }
+
+    public static MinimalServiceSettings sparseEmbedding(String serviceName) {
+        return new MinimalServiceSettings(serviceName, SPARSE_EMBEDDING, null, null, null);
     }
 
-    public static MinimalServiceSettings sparseEmbedding() {
-        return new MinimalServiceSettings(SPARSE_EMBEDDING, null, null, null);
+    public static MinimalServiceSettings rerank(String serviceName) {
+        return new MinimalServiceSettings(serviceName, RERANK, null, null, null);
     }
 
-    public static MinimalServiceSettings rerank() {
-        return new MinimalServiceSettings(RERANK, null, null, null);
+    public static MinimalServiceSettings completion(String serviceName) {
+        return new MinimalServiceSettings(serviceName, COMPLETION, null, null, null);
     }
 
-    public static MinimalServiceSettings completion() {
-        return new MinimalServiceSettings(COMPLETION, null, null, null);
+    public static MinimalServiceSettings chatCompletion(String serviceName) {
+        return new MinimalServiceSettings(serviceName, CHAT_COMPLETION, null, null, null);
     }
 
-    public static MinimalServiceSettings chatCompletion() {
-        return new MinimalServiceSettings(CHAT_COMPLETION, null, null, null);
+    public MinimalServiceSettings {
+        Objects.requireNonNull(taskType, "task type must not be null");
+        validate(taskType, dimensions, similarity, elementType);
     }
 
     public MinimalServiceSettings(Model model) {
         this(
+            model.getConfigurations().getService(),
             model.getTaskType(),
             model.getServiceSettings().dimensions(),
             model.getServiceSettings().similarity(),
             model.getServiceSettings().elementType()
         );
     }
 
-    public MinimalServiceSettings(
-        TaskType taskType,
-        @Nullable Integer dimensions,
-        @Nullable SimilarityMeasure similarity,
-        @Nullable ElementType elementType
-    ) {
-        this.taskType = Objects.requireNonNull(taskType, "task type must not be null");
-        this.dimensions = dimensions;
-        this.similarity = similarity;
-        this.elementType = elementType;
-        validate();
+    public MinimalServiceSettings(StreamInput in) throws IOException {
+        this(
+            in.readOptionalString(),
+            TaskType.fromStream(in),
+            in.readOptionalInt(),
+            in.readOptionalEnum(SimilarityMeasure.class),
+            in.readOptionalEnum(ElementType.class)
+        );
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        out.writeOptionalString(service);
+        taskType.writeTo(out);
+        out.writeOptionalInt(dimensions);
+        out.writeOptionalEnum(similarity);
+        out.writeOptionalEnum(elementType);
+    }
+
+    @Override
+    public String getWriteableName() {
+        return NAME;
+    }
+
+    @Override
+    public TransportVersion getMinimalSupportedVersion() {
+        return TransportVersions.INFERENCE_MODEL_REGISTRY_METADATA;
+    }
+
+    @Override
+    public ToXContentObject getFilteredXContentObject() {
+        return this::toXContent;
+    }
+
+    @Override
+    public String modelId() {
+        return null;
+    }
+
+    public static Diff<MinimalServiceSettings> readDiffFrom(StreamInput in) throws IOException {
+        return SimpleDiffable.readDiffFrom(MinimalServiceSettings::new, in);
     }
 
     @Override
     public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
         builder.startObject();
+        if (service != null) {
+            builder.field(SERVICE_FIELD, service);
+        }
         builder.field(TASK_TYPE_FIELD, taskType.toString());
         if (dimensions != null) {
             builder.field(DIMENSIONS_FIELD, dimensions);
@@ -143,7 +200,8 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
     @Override
     public String toString() {
         final StringBuilder sb = new StringBuilder();
-        sb.append("task_type=").append(taskType);
+        sb.append("service=").append(service);
+        sb.append(", task_type=").append(taskType);
         if (dimensions != null) {
             sb.append(", dimensions=").append(dimensions);
         }
@@ -156,31 +214,46 @@ public String toString() {
         return sb.toString();
     }
 
-    private void validate() {
+    private static void validate(TaskType taskType, Integer dimensions, SimilarityMeasure similarity, ElementType elementType) {
         switch (taskType) {
             case TEXT_EMBEDDING:
-                validateFieldPresent(DIMENSIONS_FIELD, dimensions);
-                validateFieldPresent(SIMILARITY_FIELD, similarity);
-                validateFieldPresent(ELEMENT_TYPE_FIELD, elementType);
+                validateFieldPresent(DIMENSIONS_FIELD, dimensions, taskType);
+                validateFieldPresent(SIMILARITY_FIELD, similarity, taskType);
+                validateFieldPresent(ELEMENT_TYPE_FIELD, elementType, taskType);
                 break;
 
             default:
-                validateFieldNotPresent(DIMENSIONS_FIELD, dimensions);
-                validateFieldNotPresent(SIMILARITY_FIELD, similarity);
-                validateFieldNotPresent(ELEMENT_TYPE_FIELD, elementType);
+                validateFieldNotPresent(DIMENSIONS_FIELD, dimensions, taskType);
+                validateFieldNotPresent(SIMILARITY_FIELD, similarity, taskType);
+                validateFieldNotPresent(ELEMENT_TYPE_FIELD, elementType, taskType);
                 break;
         }
     }
 
-    private void validateFieldPresent(String field, Object fieldValue) {
+    private static void validateFieldPresent(String field, Object fieldValue, TaskType taskType) {
         if (fieldValue == null) {
             throw new IllegalArgumentException("required [" + field + "] field is missing for task_type [" + taskType.name() + "]");
         }
     }
 
-    private void validateFieldNotPresent(String field, Object fieldValue) {
+    private static void validateFieldNotPresent(String field, Object fieldValue, TaskType taskType) {
         if (fieldValue != null) {
             throw new IllegalArgumentException("[" + field + "] is not allowed for task_type [" + taskType.name() + "]");
         }
     }
+
+    public ModelConfigurations toModelConfigurations(String inferenceEntityId) {
+        return new ModelConfigurations(inferenceEntityId, taskType, service == null ? UNKNOWN_SERVICE : service, this);
+    }
+
+    /**
+     * Checks if the given {@link MinimalServiceSettings} is equivalent to the current definition.
+     */
+    public boolean canMergeWith(MinimalServiceSettings other) {
+        return taskType == other.taskType
+            && Objects.equals(dimensions, other.dimensions)
+            && similarity == other.similarity
+            && elementType == other.elementType
+            && (service == null || service.equals(other.service));
+    }
 }

server/src/test/java/org/elasticsearch/inference/MinimalServiceSettingsTests.java

@@ -16,8 +16,7 @@
 import java.io.IOException;
 
 public class MinimalServiceSettingsTests extends AbstractXContentTestCase<MinimalServiceSettings> {
-    @Override
-    protected MinimalServiceSettings createTestInstance() {
+    public static MinimalServiceSettings randomInstance() {
         TaskType taskType = randomFrom(TaskType.values());
         Integer dimensions = null;
         SimilarityMeasure similarity = null;
@@ -28,7 +27,12 @@ protected MinimalServiceSettings createTestInstance() {
             similarity = randomFrom(SimilarityMeasure.values());
             elementType = randomFrom(DenseVectorFieldMapper.ElementType.values());
         }
-        return new MinimalServiceSettings(taskType, dimensions, similarity, elementType);
+        return new MinimalServiceSettings(randomBoolean() ? null : randomAlphaOfLength(10), taskType, dimensions, similarity, elementType);
+    }
+
+    @Override
+    protected MinimalServiceSettings createTestInstance() {
+        return randomInstance();
     }
 
     @Override

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/action/GetInferenceModelAction.java

@@ -44,18 +44,23 @@ public static class Request extends AcknowledgedRequest<GetInferenceModelAction.
         // no effect when getting a single model
         private final boolean persistDefaultConfig;
 
+        // For testing only, retrieves the minimal config from the cluster state.
+        private final boolean returnMinimalConfig;
+
         public Request(String inferenceEntityId, TaskType taskType) {
-            super(TRAPPY_IMPLICIT_DEFAULT_MASTER_NODE_TIMEOUT, DEFAULT_ACK_TIMEOUT);
-            this.inferenceEntityId = Objects.requireNonNull(inferenceEntityId);
-            this.taskType = Objects.requireNonNull(taskType);
-            this.persistDefaultConfig = PERSIST_DEFAULT_CONFIGS;
+            this(inferenceEntityId, taskType, PERSIST_DEFAULT_CONFIGS);
         }
 
         public Request(String inferenceEntityId, TaskType taskType, boolean persistDefaultConfig) {
+            this(inferenceEntityId, taskType, persistDefaultConfig, false);
+        }
+
+        public Request(String inferenceEntityId, TaskType taskType, boolean persistDefaultConfig, boolean returnMinimalConfig) {
             super(TRAPPY_IMPLICIT_DEFAULT_MASTER_NODE_TIMEOUT, DEFAULT_ACK_TIMEOUT);
             this.inferenceEntityId = Objects.requireNonNull(inferenceEntityId);
             this.taskType = Objects.requireNonNull(taskType);
             this.persistDefaultConfig = persistDefaultConfig;
+            this.returnMinimalConfig = returnMinimalConfig;
         }
 
         public Request(StreamInput in) throws IOException {
@@ -68,6 +73,12 @@ public Request(StreamInput in) throws IOException {
                 this.persistDefaultConfig = PERSIST_DEFAULT_CONFIGS;
             }
 
+            if (in.getTransportVersion().onOrAfter(TransportVersions.INFERENCE_MODEL_REGISTRY_METADATA)) {
+                this.returnMinimalConfig = in.readBoolean();
+            } else {
+                this.returnMinimalConfig = false;
+            }
+
         }
 
         public String getInferenceEntityId() {
@@ -82,6 +93,10 @@ public boolean isPersistDefaultConfig() {
             return persistDefaultConfig;
         }
 
+        public boolean isReturnMinimalConfig() {
+            return returnMinimalConfig;
+        }
+
         @Override
         public void writeTo(StreamOutput out) throws IOException {
             super.writeTo(out);
@@ -90,6 +105,10 @@ public void writeTo(StreamOutput out) throws IOException {
             if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_16_0)) {
                 out.writeBoolean(this.persistDefaultConfig);
             }
+
+            if (out.getTransportVersion().onOrAfter(TransportVersions.INFERENCE_MODEL_REGISTRY_METADATA)) {
+                out.writeBoolean(returnMinimalConfig);
+            }
         }
 
         @Override
@@ -99,12 +118,13 @@ public boolean equals(Object o) {
             Request request = (Request) o;
             return Objects.equals(inferenceEntityId, request.inferenceEntityId)
                 && taskType == request.taskType
-                && persistDefaultConfig == request.persistDefaultConfig;
+                && persistDefaultConfig == request.persistDefaultConfig
+                && returnMinimalConfig == request.returnMinimalConfig;
         }
 
         @Override
         public int hashCode() {
-            return Objects.hash(inferenceEntityId, taskType, persistDefaultConfig);
+            return Objects.hash(inferenceEntityId, taskType, persistDefaultConfig, returnMinimalConfig);
         }
     }
 

x-pack/plugin/esql/qa/server/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/multi_node/SemanticMatchIT.java

@@ -17,7 +17,9 @@
 @ThreadLeakFilters(filters = TestClustersThreadFilter.class)
 public class SemanticMatchIT extends SemanticMatchTestCase {
     @ClassRule
-    public static ElasticsearchCluster cluster = Clusters.testCluster(spec -> spec.plugin("inference-service-test"));
+    public static ElasticsearchCluster cluster = Clusters.testCluster(
+        spec -> spec.module("x-pack-inference").plugin("inference-service-test")
+    );
 
     @Override
     protected String getTestRestCluster() {