Skip to content

Commit 3a5feb1

Browse files
authored
Merge branch 'main' into arpadkiraly-remove-updateforv9-entirely
2 parents b04708f + 9f76a7f commit 3a5feb1

File tree

45 files changed

+1528
-647
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

45 files changed

+1528
-647
lines changed

docs/changelog/121106.yaml

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
pr: 121106
2+
summary: Add `ModelRegistryMetadata` to Cluster State
3+
area: Machine Learning
4+
type: enhancement
5+
issues: []

libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/PolicyUtils.java

Lines changed: 17 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -49,29 +49,34 @@ public record PluginData(Path pluginPath, boolean isModular, boolean isExternalP
4949

5050
private static final String POLICY_FILE_NAME = "entitlement-policy.yaml";
5151

52-
public static Map<String, Policy> createPluginPolicies(Collection<PluginData> pluginData, Map<String, String> overrides, String version)
53-
throws IOException {
52+
public static Map<String, Policy> createPluginPolicies(
53+
Collection<PluginData> pluginData,
54+
Map<String, String> pluginPolicyPatches,
55+
String version
56+
) throws IOException {
5457
Map<String, Policy> pluginPolicies = new HashMap<>(pluginData.size());
5558
for (var entry : pluginData) {
5659
Path pluginRoot = entry.pluginPath();
60+
Path policyFile = pluginRoot.resolve(POLICY_FILE_NAME);
5761
String pluginName = pluginRoot.getFileName().toString();
5862
final Set<String> moduleNames = getModuleNames(pluginRoot, entry.isModular());
5963

60-
var overriddenPolicy = parseEncodedPolicyIfExists(
61-
overrides.get(pluginName),
64+
var pluginPolicyPatch = parseEncodedPolicyIfExists(
65+
pluginPolicyPatches.get(pluginName),
6266
version,
6367
entry.isExternalPlugin(),
6468
pluginName,
6569
moduleNames
6670
);
67-
if (overriddenPolicy != null) {
68-
pluginPolicies.put(pluginName, overriddenPolicy);
69-
} else {
70-
Path policyFile = pluginRoot.resolve(POLICY_FILE_NAME);
71-
var policy = parsePolicyIfExists(pluginName, policyFile, entry.isExternalPlugin());
72-
validatePolicyScopes(pluginName, policy, moduleNames, policyFile.toString());
73-
pluginPolicies.put(pluginName, policy);
74-
}
71+
var pluginPolicy = parsePolicyIfExists(pluginName, policyFile, entry.isExternalPlugin());
72+
validatePolicyScopes(pluginName, pluginPolicy, moduleNames, policyFile.toString());
73+
74+
pluginPolicies.put(
75+
pluginName,
76+
pluginPolicyPatch == null
77+
? pluginPolicy
78+
: new Policy(pluginPolicy.name(), PolicyUtils.mergeScopes(pluginPolicy.scopes(), pluginPolicyPatch.scopes()))
79+
);
7580
}
7681
return pluginPolicies;
7782
}

muted-tests.yml

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -408,6 +408,9 @@ tests:
408408
- class: org.elasticsearch.xpack.inference.external.request.azureopenai.embeddings.AzureOpenAiEmbeddingsRequestTests
409409
method: testCreateRequest_WithEntraIdDefined
410410
issue: https://github.com/elastic/elasticsearch/issues/125061
411+
- class: org.elasticsearch.xpack.ilm.DataStreamAndIndexLifecycleMixingTests
412+
method: testGetDataStreamResponse
413+
issue: https://github.com/elastic/elasticsearch/issues/125083
411414

412415
# Examples:
413416
#

server/src/main/java/org/elasticsearch/TransportVersions.java

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -187,6 +187,7 @@ static TransportVersion def(int id) {
187187
public static final TransportVersion ML_INFERENCE_DEEPSEEK = def(9_029_00_0);
188188
public static final TransportVersion ESQL_FAILURE_FROM_REMOTE = def(9_030_00_0);
189189
public static final TransportVersion INDEX_RESHARDING_METADATA = def(9_031_0_00);
190+
public static final TransportVersion INFERENCE_MODEL_REGISTRY_METADATA = def(9_032_0_00);
190191

191192
/*
192193
* STOP! READ THIS FIRST! No, really,

server/src/main/java/org/elasticsearch/bootstrap/Elasticsearch.java

Lines changed: 17 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -86,8 +86,8 @@
8686
*/
8787
class Elasticsearch {
8888

89-
private static final String PLUGIN_POLICY_OVERRIDE_PREFIX = "es.entitlements.policy.";
90-
private static final String SERVER_POLICY_OVERRIDE = "es.entitlements.server_policy";
89+
private static final String POLICY_PATCH_PREFIX = "es.entitlements.policy.";
90+
private static final String SERVER_POLICY_PATCH_NAME = POLICY_PATCH_PREFIX + "server";
9191

9292
/**
9393
* Main entry point for starting elasticsearch.
@@ -253,10 +253,10 @@ private static void initPhase2(Bootstrap bootstrap) throws IOException {
253253
.map(bundle -> new PolicyUtils.PluginData(bundle.getDir(), bundle.pluginDescriptor().isModular(), true))
254254
).toList();
255255

256-
var pluginPolicyOverrides = collectPluginPolicyOverrides(modulesBundles, pluginsBundles, logger);
257-
var pluginPolicies = PolicyUtils.createPluginPolicies(pluginData, pluginPolicyOverrides, Build.current().version());
256+
var pluginPolicyPatches = collectPluginPolicyPatches(modulesBundles, pluginsBundles, logger);
257+
var pluginPolicies = PolicyUtils.createPluginPolicies(pluginData, pluginPolicyPatches, Build.current().version());
258258
var serverPolicyPatch = PolicyUtils.parseEncodedPolicyIfExists(
259-
System.getProperty(SERVER_POLICY_OVERRIDE),
259+
System.getProperty(SERVER_POLICY_PATCH_NAME),
260260
Build.current().version(),
261261
false,
262262
"server",
@@ -331,33 +331,36 @@ private static void logSystemInfo() {
331331
}
332332
}
333333

334-
private static Map<String, String> collectPluginPolicyOverrides(
334+
private static Map<String, String> collectPluginPolicyPatches(
335335
Set<PluginBundle> modulesBundles,
336336
Set<PluginBundle> pluginsBundles,
337337
Logger logger
338338
) {
339-
var policyOverrides = new HashMap<String, String>();
339+
var policyPatches = new HashMap<String, String>();
340340
var systemProperties = BootstrapInfo.getSystemProperties();
341341
systemProperties.keys().asIterator().forEachRemaining(key -> {
342342
var value = systemProperties.get(key);
343-
if (key instanceof String k && k.startsWith(PLUGIN_POLICY_OVERRIDE_PREFIX) && value instanceof String v) {
344-
policyOverrides.put(k.substring(PLUGIN_POLICY_OVERRIDE_PREFIX.length()), v);
343+
if (key instanceof String k
344+
&& value instanceof String v
345+
&& k.startsWith(POLICY_PATCH_PREFIX)
346+
&& k.equals(SERVER_POLICY_PATCH_NAME) == false) {
347+
policyPatches.put(k.substring(POLICY_PATCH_PREFIX.length()), v);
345348
}
346349
});
347350
var pluginNames = Stream.concat(modulesBundles.stream(), pluginsBundles.stream())
348351
.map(bundle -> bundle.pluginDescriptor().getName())
349352
.collect(Collectors.toUnmodifiableSet());
350353

351-
for (var overriddenPluginName : policyOverrides.keySet()) {
352-
if (pluginNames.contains(overriddenPluginName) == false) {
354+
for (var patchedPluginName : policyPatches.keySet()) {
355+
if (pluginNames.contains(patchedPluginName) == false) {
353356
logger.warn(
354-
"Found command-line override for unknown plugin [{}] (available plugins: [{}])",
355-
overriddenPluginName,
357+
"Found command-line policy patch for unknown plugin [{}] (available plugins: [{}])",
358+
patchedPluginName,
356359
String.join(", ", pluginNames)
357360
);
358361
}
359362
}
360-
return policyOverrides;
363+
return policyPatches;
361364
}
362365

363366
private static class EntitlementSelfTester {

server/src/main/java/org/elasticsearch/inference/MinimalServiceSettings.java

Lines changed: 111 additions & 38 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,12 @@
99

1010
package org.elasticsearch.inference;
1111

12+
import org.elasticsearch.TransportVersion;
13+
import org.elasticsearch.TransportVersions;
14+
import org.elasticsearch.cluster.Diff;
15+
import org.elasticsearch.cluster.SimpleDiffable;
16+
import org.elasticsearch.common.io.stream.StreamInput;
17+
import org.elasticsearch.common.io.stream.StreamOutput;
1218
import org.elasticsearch.core.Nullable;
1319
import org.elasticsearch.index.mapper.vectors.DenseVectorFieldMapper;
1420
import org.elasticsearch.xcontent.ConstructingObjectParser;
@@ -46,12 +52,16 @@
4652
* @param elementType the type of elements in the embeddings, applicable only for {@link TaskType#TEXT_EMBEDDING} (nullable).
4753
*/
4854
public record MinimalServiceSettings(
55+
@Nullable String service,
4956
TaskType taskType,
5057
@Nullable Integer dimensions,
5158
@Nullable SimilarityMeasure similarity,
5259
@Nullable ElementType elementType
53-
) implements ToXContentObject {
60+
) implements ServiceSettings, SimpleDiffable<MinimalServiceSettings> {
5461

62+
public static final String NAME = "minimal_service_settings";
63+
64+
public static final String SERVICE_FIELD = "service";
5565
public static final String TASK_TYPE_FIELD = "task_type";
5666
static final String DIMENSIONS_FIELD = "dimensions";
5767
static final String SIMILARITY_FIELD = "similarity";
@@ -61,17 +71,20 @@ public record MinimalServiceSettings(
6171
"model_settings",
6272
true,
6373
args -> {
64-
TaskType taskType = TaskType.fromString((String) args[0]);
65-
Integer dimensions = (Integer) args[1];
66-
SimilarityMeasure similarity = args[2] == null ? null : SimilarityMeasure.fromString((String) args[2]);
67-
DenseVectorFieldMapper.ElementType elementType = args[3] == null
74+
String service = (String) args[0];
75+
TaskType taskType = TaskType.fromString((String) args[1]);
76+
Integer dimensions = (Integer) args[2];
77+
SimilarityMeasure similarity = args[3] == null ? null : SimilarityMeasure.fromString((String) args[3]);
78+
DenseVectorFieldMapper.ElementType elementType = args[4] == null
6879
? null
69-
: DenseVectorFieldMapper.ElementType.fromString((String) args[3]);
70-
return new MinimalServiceSettings(taskType, dimensions, similarity, elementType);
80+
: DenseVectorFieldMapper.ElementType.fromString((String) args[4]);
81+
return new MinimalServiceSettings(service, taskType, dimensions, similarity, elementType);
7182
}
7283
);
84+
private static final String UNKNOWN_SERVICE = "_unknown_";
7385

7486
static {
87+
PARSER.declareString(ConstructingObjectParser.optionalConstructorArg(), new ParseField(SERVICE_FIELD));
7588
PARSER.declareString(ConstructingObjectParser.constructorArg(), new ParseField(TASK_TYPE_FIELD));
7689
PARSER.declareInt(ConstructingObjectParser.optionalConstructorArg(), new ParseField(DIMENSIONS_FIELD));
7790
PARSER.declareString(ConstructingObjectParser.optionalConstructorArg(), new ParseField(SIMILARITY_FIELD));
@@ -82,51 +95,95 @@ public static MinimalServiceSettings parse(XContentParser parser) throws IOExcep
8295
return PARSER.parse(parser, null);
8396
}
8497

85-
public static MinimalServiceSettings textEmbedding(int dimensions, SimilarityMeasure similarity, ElementType elementType) {
86-
return new MinimalServiceSettings(TEXT_EMBEDDING, dimensions, similarity, elementType);
98+
public static MinimalServiceSettings textEmbedding(
99+
String serviceName,
100+
int dimensions,
101+
SimilarityMeasure similarity,
102+
ElementType elementType
103+
) {
104+
return new MinimalServiceSettings(serviceName, TEXT_EMBEDDING, dimensions, similarity, elementType);
105+
}
106+
107+
public static MinimalServiceSettings sparseEmbedding(String serviceName) {
108+
return new MinimalServiceSettings(serviceName, SPARSE_EMBEDDING, null, null, null);
87109
}
88110

89-
public static MinimalServiceSettings sparseEmbedding() {
90-
return new MinimalServiceSettings(SPARSE_EMBEDDING, null, null, null);
111+
public static MinimalServiceSettings rerank(String serviceName) {
112+
return new MinimalServiceSettings(serviceName, RERANK, null, null, null);
91113
}
92114

93-
public static MinimalServiceSettings rerank() {
94-
return new MinimalServiceSettings(RERANK, null, null, null);
115+
public static MinimalServiceSettings completion(String serviceName) {
116+
return new MinimalServiceSettings(serviceName, COMPLETION, null, null, null);
95117
}
96118

97-
public static MinimalServiceSettings completion() {
98-
return new MinimalServiceSettings(COMPLETION, null, null, null);
119+
public static MinimalServiceSettings chatCompletion(String serviceName) {
120+
return new MinimalServiceSettings(serviceName, CHAT_COMPLETION, null, null, null);
99121
}
100122

101-
public static MinimalServiceSettings chatCompletion() {
102-
return new MinimalServiceSettings(CHAT_COMPLETION, null, null, null);
123+
public MinimalServiceSettings {
124+
Objects.requireNonNull(taskType, "task type must not be null");
125+
validate(taskType, dimensions, similarity, elementType);
103126
}
104127

105128
public MinimalServiceSettings(Model model) {
106129
this(
130+
model.getConfigurations().getService(),
107131
model.getTaskType(),
108132
model.getServiceSettings().dimensions(),
109133
model.getServiceSettings().similarity(),
110134
model.getServiceSettings().elementType()
111135
);
112136
}
113137

114-
public MinimalServiceSettings(
115-
TaskType taskType,
116-
@Nullable Integer dimensions,
117-
@Nullable SimilarityMeasure similarity,
118-
@Nullable ElementType elementType
119-
) {
120-
this.taskType = Objects.requireNonNull(taskType, "task type must not be null");
121-
this.dimensions = dimensions;
122-
this.similarity = similarity;
123-
this.elementType = elementType;
124-
validate();
138+
public MinimalServiceSettings(StreamInput in) throws IOException {
139+
this(
140+
in.readOptionalString(),
141+
TaskType.fromStream(in),
142+
in.readOptionalInt(),
143+
in.readOptionalEnum(SimilarityMeasure.class),
144+
in.readOptionalEnum(ElementType.class)
145+
);
146+
}
147+
148+
@Override
149+
public void writeTo(StreamOutput out) throws IOException {
150+
out.writeOptionalString(service);
151+
taskType.writeTo(out);
152+
out.writeOptionalInt(dimensions);
153+
out.writeOptionalEnum(similarity);
154+
out.writeOptionalEnum(elementType);
155+
}
156+
157+
@Override
158+
public String getWriteableName() {
159+
return NAME;
160+
}
161+
162+
@Override
163+
public TransportVersion getMinimalSupportedVersion() {
164+
return TransportVersions.INFERENCE_MODEL_REGISTRY_METADATA;
165+
}
166+
167+
@Override
168+
public ToXContentObject getFilteredXContentObject() {
169+
return this::toXContent;
170+
}
171+
172+
@Override
173+
public String modelId() {
174+
return null;
175+
}
176+
177+
public static Diff<MinimalServiceSettings> readDiffFrom(StreamInput in) throws IOException {
178+
return SimpleDiffable.readDiffFrom(MinimalServiceSettings::new, in);
125179
}
126180

127181
@Override
128182
public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
129183
builder.startObject();
184+
if (service != null) {
185+
builder.field(SERVICE_FIELD, service);
186+
}
130187
builder.field(TASK_TYPE_FIELD, taskType.toString());
131188
if (dimensions != null) {
132189
builder.field(DIMENSIONS_FIELD, dimensions);
@@ -143,7 +200,8 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
143200
@Override
144201
public String toString() {
145202
final StringBuilder sb = new StringBuilder();
146-
sb.append("task_type=").append(taskType);
203+
sb.append("service=").append(service);
204+
sb.append(", task_type=").append(taskType);
147205
if (dimensions != null) {
148206
sb.append(", dimensions=").append(dimensions);
149207
}
@@ -156,31 +214,46 @@ public String toString() {
156214
return sb.toString();
157215
}
158216

159-
private void validate() {
217+
private static void validate(TaskType taskType, Integer dimensions, SimilarityMeasure similarity, ElementType elementType) {
160218
switch (taskType) {
161219
case TEXT_EMBEDDING:
162-
validateFieldPresent(DIMENSIONS_FIELD, dimensions);
163-
validateFieldPresent(SIMILARITY_FIELD, similarity);
164-
validateFieldPresent(ELEMENT_TYPE_FIELD, elementType);
220+
validateFieldPresent(DIMENSIONS_FIELD, dimensions, taskType);
221+
validateFieldPresent(SIMILARITY_FIELD, similarity, taskType);
222+
validateFieldPresent(ELEMENT_TYPE_FIELD, elementType, taskType);
165223
break;
166224

167225
default:
168-
validateFieldNotPresent(DIMENSIONS_FIELD, dimensions);
169-
validateFieldNotPresent(SIMILARITY_FIELD, similarity);
170-
validateFieldNotPresent(ELEMENT_TYPE_FIELD, elementType);
226+
validateFieldNotPresent(DIMENSIONS_FIELD, dimensions, taskType);
227+
validateFieldNotPresent(SIMILARITY_FIELD, similarity, taskType);
228+
validateFieldNotPresent(ELEMENT_TYPE_FIELD, elementType, taskType);
171229
break;
172230
}
173231
}
174232

175-
private void validateFieldPresent(String field, Object fieldValue) {
233+
private static void validateFieldPresent(String field, Object fieldValue, TaskType taskType) {
176234
if (fieldValue == null) {
177235
throw new IllegalArgumentException("required [" + field + "] field is missing for task_type [" + taskType.name() + "]");
178236
}
179237
}
180238

181-
private void validateFieldNotPresent(String field, Object fieldValue) {
239+
private static void validateFieldNotPresent(String field, Object fieldValue, TaskType taskType) {
182240
if (fieldValue != null) {
183241
throw new IllegalArgumentException("[" + field + "] is not allowed for task_type [" + taskType.name() + "]");
184242
}
185243
}
244+
245+
public ModelConfigurations toModelConfigurations(String inferenceEntityId) {
246+
return new ModelConfigurations(inferenceEntityId, taskType, service == null ? UNKNOWN_SERVICE : service, this);
247+
}
248+
249+
/**
250+
* Checks if the given {@link MinimalServiceSettings} is equivalent to the current definition.
251+
*/
252+
public boolean canMergeWith(MinimalServiceSettings other) {
253+
return taskType == other.taskType
254+
&& Objects.equals(dimensions, other.dimensions)
255+
&& similarity == other.similarity
256+
&& elementType == other.elementType
257+
&& (service == null || service.equals(other.service));
258+
}
186259
}

0 commit comments

Comments
 (0)