Skip to content

Commit d1a042b

Browse files
ldematteldematte
authored
Merge branch 'main' into entitlements/file-system-provider
2 parents 04ab43d + b9d1222 commit d1a042b

File tree

13 files changed

+125
-41
lines changed

13 files changed

+125
-41
lines changed

‎docs/changelog/122066.yaml‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
pr: 122066
2+
summary: Adding elser default endpoint for EIS
3+
area: Machine Learning
4+
type: enhancement
5+
issues: []

‎docs/changelog/122257.yaml‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
pr: 122257
2+
summary: Revive inlinestats
3+
area: ES|QL
4+
type: bug
5+
issues: []

‎x-pack/plugin/esql/qa/server/multi-clusters/src/javaRestTest/java/org/elasticsearch/xpack/esql/ccq/MultiClusterSpecIT.java‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -48,6 +48,7 @@
4848
import static org.elasticsearch.xpack.esql.EsqlTestUtils.classpathResources;
4949
import static org.elasticsearch.xpack.esql.action.EsqlCapabilities.Cap.INLINESTATS;
5050
import static org.elasticsearch.xpack.esql.action.EsqlCapabilities.Cap.INLINESTATS_V2;
51+
import static org.elasticsearch.xpack.esql.action.EsqlCapabilities.Cap.INLINESTATS_V3;
5152
import static org.elasticsearch.xpack.esql.action.EsqlCapabilities.Cap.JOIN_LOOKUP_V12;
5253
import static org.elasticsearch.xpack.esql.action.EsqlCapabilities.Cap.JOIN_PLANNING_V1;
5354
import static org.elasticsearch.xpack.esql.action.EsqlCapabilities.Cap.METADATA_FIELDS_REMOTE_TEST;
@@ -124,6 +125,7 @@ protected void shouldSkipTest(String testName) throws IOException {
124125
assumeFalse("INLINESTATS not yet supported in CCS", testCase.requiredCapabilities.contains(INLINESTATS.capabilityName()));
125126
assumeFalse("INLINESTATS not yet supported in CCS", testCase.requiredCapabilities.contains(INLINESTATS_V2.capabilityName()));
126127
assumeFalse("INLINESTATS not yet supported in CCS", testCase.requiredCapabilities.contains(JOIN_PLANNING_V1.capabilityName()));
128+
assumeFalse("INLINESTATS not yet supported in CCS", testCase.requiredCapabilities.contains(INLINESTATS_V3.capabilityName()));
127129
assumeFalse("LOOKUP JOIN not yet supported in CCS", testCase.requiredCapabilities.contains(JOIN_LOOKUP_V12.capabilityName()));
128130
}
129131

‎x-pack/plugin/esql/qa/testFixtures/src/main/resources/inlinestats.csv-spec-ignored‎ renamed to ‎x-pack/plugin/esql/qa/testFixtures/src/main/resources/inlinestats.csv-spec‎

Lines changed: 22 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -2,8 +2,8 @@
22
// TODO: re-enable the commented tests once the Join functionality stabilizes
33
//
44

5-
maxOfInt-Ignore
6-
required_capability: join_planning_v1
5+
maxOfInt
6+
required_capability: inlinestats_v3
77
// tag::max-languages[]
88
FROM employees
99
| KEEP emp_no, languages
@@ -25,7 +25,7 @@ emp_no:integer | languages:integer | max_lang:integer
2525
;
2626

2727
maxOfIntByKeyword
28-
required_capability: join_planning_v1
28+
required_capability: inlinestats_v3
2929

3030
FROM employees
3131
| KEEP emp_no, languages, gender
@@ -43,7 +43,7 @@ emp_no:integer | languages:integer | gender:keyword | max_lang:integer
4343
;
4444

4545
maxOfLongByKeyword
46-
required_capability: join_planning_v1
46+
required_capability: inlinestats_v3
4747

4848
FROM employees
4949
| KEEP emp_no, avg_worked_seconds, gender
@@ -57,8 +57,8 @@ emp_no:integer | avg_worked_seconds:long | gender:keyword | max_avg_worked_secon
5757
10030 | 394597613 | M | 394597613
5858
;
5959

60-
maxOfLong-Ignore
61-
required_capability: join_planning_v1
60+
maxOfLong
61+
required_capability: inlinestats_v3
6262

6363
FROM employees
6464
| KEEP emp_no, avg_worked_seconds, gender
@@ -71,7 +71,7 @@ emp_no:integer | avg_worked_seconds:long | gender:keyword | max_avg_worked_secon
7171
;
7272

7373
maxOfLongByCalculatedKeyword
74-
required_capability: join_planning_v1
74+
required_capability: inlinestats_v3
7575

7676
// tag::longest-tenured-by-first[]
7777
FROM employees
@@ -94,7 +94,7 @@ emp_no:integer | avg_worked_seconds:long | last_name:keyword | SUBSTRING(last_na
9494
;
9595

9696
maxOfLongByCalculatedNamedKeyword
97-
required_capability: join_planning_v1
97+
required_capability: inlinestats_v3
9898

9999
FROM employees
100100
| KEEP emp_no, avg_worked_seconds, last_name
@@ -112,7 +112,7 @@ emp_no:integer | avg_worked_seconds:long | last_name:keyword | l:keyword | max_a
112112
10087 | 305782871 | Eugenio | E | 305782871
113113
;
114114

115-
maxOfLongByCalculatedDroppedKeyword
115+
maxOfLongByCalculatedDroppedKeyword-Ignore
116116
required_capability: join_planning_v1
117117

118118
FROM employees
@@ -132,7 +132,7 @@ emp_no:integer | avg_worked_seconds:long | last_name:keyword | max_avg_worked_se
132132
;
133133

134134
maxOfLongByEvaledKeyword
135-
required_capability: join_planning_v1
135+
required_capability: inlinestats_v3
136136

137137
FROM employees
138138
| EVAL l = SUBSTRING(last_name, 0, 1)
@@ -152,7 +152,7 @@ emp_no:integer | avg_worked_seconds:long | l:keyword | max_avg_worked_seconds:lo
152152
;
153153

154154
maxOfLongByInt
155-
required_capability: join_planning_v1
155+
required_capability: inlinestats_v3
156156

157157
FROM employees
158158
| KEEP emp_no, avg_worked_seconds, languages
@@ -170,7 +170,7 @@ emp_no:integer | avg_worked_seconds:long | languages:integer | max_avg_worked_se
170170
;
171171

172172
maxOfLongByIntDouble
173-
required_capability: join_planning_v1
173+
required_capability: inlinestats_v3
174174

175175
FROM employees
176176
| KEEP emp_no, avg_worked_seconds, languages, height
@@ -205,7 +205,7 @@ emp_no:integer | languages:integer | avg_worked_seconds:long | gender:keyword |
205205
10007 | 4 | 393084805 | F | 2.863684210555556E8 | 5
206206
;
207207

208-
byMultivaluedSimple
208+
byMultivaluedSimple-Ignore
209209
required_capability: join_planning_v1
210210

211211
// tag::mv-group[]
@@ -223,7 +223,7 @@ abbrev:keyword | type:keyword | scalerank:integer | min_scalerank:integer
223223
// end::mv-group-result[]
224224
;
225225

226-
byMultivaluedMvExpand
226+
byMultivaluedMvExpand-Ignore
227227
required_capability: join_planning_v1
228228

229229
// tag::mv-expand[]
@@ -243,7 +243,7 @@ abbrev:keyword | type:keyword | scalerank:integer | min_scalerank:integer
243243
// end::mv-expand-result[]
244244
;
245245

246-
byMvExpand
246+
byMvExpand-Ignore
247247
required_capability: join_planning_v1
248248

249249
// tag::extreme-airports[]
@@ -307,7 +307,7 @@ count:long | country:keyword | avg:double
307307
17 | United Kingdom | 4.455
308308
;
309309

310-
afterWhere
310+
afterWhere-Ignore
311311
required_capability: join_planning_v1
312312

313313
FROM airports
@@ -366,8 +366,8 @@ abbrev:keyword | city:keyword | region:text | "COUNT(*)":long
366366
FUK | Fukuoka | 中央区 | 2
367367
;
368368

369-
beforeStats-Ignore
370-
required_capability: join_planning_v1
369+
beforeStats
370+
required_capability: inlinestats_v3
371371

372372
FROM airports
373373
| EVAL lat = ST_Y(location)
@@ -379,7 +379,7 @@ northern:long | southern:long
379379
520 | 371
380380
;
381381

382-
beforeKeepSort
382+
beforeKeepSort-Ignore
383383
required_capability: join_planning_v1
384384

385385
FROM employees
@@ -394,7 +394,7 @@ emp_no:integer | languages:integer | max_salary:integer
394394
10003 | 4 | 74572
395395
;
396396

397-
beforeKeepWhere
397+
beforeKeepWhere-Ignore
398398
required_capability: join_planning_v1
399399

400400
FROM employees
@@ -537,8 +537,8 @@ emp_no:integer | one:integer
537537
10005 | 1
538538
;
539539

540-
percentile-Ignore
541-
required_capability: join_planning_v1
540+
percentile
541+
required_capability: inlinestats_v3
542542

543543
FROM employees
544544
| KEEP emp_no, salary

‎x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -808,7 +808,13 @@ public enum Cap {
808808
* and https://github.com/elastic/elasticsearch/issues/120803
809809
* Support for queries that have multiple SORTs that cannot become TopN
810810
*/
811-
REMOVE_REDUNDANT_SORT;
811+
REMOVE_REDUNDANT_SORT,
812+
813+
/**
814+
* Fixes a series of issues with inlinestats which had an incomplete implementation after lookup and inlinestats
815+
* were refactored.
816+
*/
817+
INLINESTATS_V3;
812818

813819
private final boolean enabled;
814820

‎x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/join/InlineJoin.java‎

Lines changed: 3 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -16,10 +16,9 @@
1616
import org.elasticsearch.xpack.esql.core.expression.Literal;
1717
import org.elasticsearch.xpack.esql.core.tree.NodeInfo;
1818
import org.elasticsearch.xpack.esql.core.tree.Source;
19-
import org.elasticsearch.xpack.esql.core.util.CollectionUtils;
2019
import org.elasticsearch.xpack.esql.io.stream.PlanStreamInput;
20+
import org.elasticsearch.xpack.esql.plan.logical.Eval;
2121
import org.elasticsearch.xpack.esql.plan.logical.LogicalPlan;
22-
import org.elasticsearch.xpack.esql.plan.logical.Project;
2322
import org.elasticsearch.xpack.esql.plan.logical.UnaryPlan;
2423
import org.elasticsearch.xpack.esql.plan.logical.local.LocalRelation;
2524

@@ -71,10 +70,9 @@ public static LogicalPlan inlineData(InlineJoin target, LocalRelation data) {
7170
List<Alias> aliases = new ArrayList<>(schema.size());
7271
for (int i = 0; i < schema.size(); i++) {
7372
Attribute attr = schema.get(i);
74-
aliases.add(new Alias(attr.source(), attr.name(), Literal.of(attr, BlockUtils.toJavaObject(blocks[i], 0))));
73+
aliases.add(new Alias(attr.source(), attr.name(), Literal.of(attr, BlockUtils.toJavaObject(blocks[i], 0)), attr.id()));
7574
}
76-
LogicalPlan left = target.left();
77-
return new Project(target.source(), left, CollectionUtils.combine(left.output(), aliases));
75+
return new Eval(target.source(), target.left(), aliases);
7876
} else {
7977
return target.replaceRight(data);
8078
}

‎x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/mapper/Mapper.java‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@
2222
import org.elasticsearch.xpack.esql.plan.logical.OrderBy;
2323
import org.elasticsearch.xpack.esql.plan.logical.TopN;
2424
import org.elasticsearch.xpack.esql.plan.logical.UnaryPlan;
25+
import org.elasticsearch.xpack.esql.plan.logical.join.InlineJoin;
2526
import org.elasticsearch.xpack.esql.plan.logical.join.Join;
2627
import org.elasticsearch.xpack.esql.plan.logical.join.JoinConfig;
2728
import org.elasticsearch.xpack.esql.plan.logical.join.JoinTypes;
@@ -178,6 +179,10 @@ private PhysicalPlan mapBinary(BinaryPlan bp) {
178179
throw new EsqlIllegalArgumentException("unsupported join type [" + config.type() + "]");
179180
}
180181

182+
if (join instanceof InlineJoin) {
183+
return new FragmentExec(bp);
184+
}
185+
181186
PhysicalPlan left = map(bp.left());
182187

183188
// only broadcast joins supported for now - hence push down as a streaming operator

‎x-pack/plugin/inference/qa/inference-service-tests/src/javaRestTest/java/org/elasticsearch/xpack/inference/InferenceGetModelsWithElasticInferenceServiceIT.java‎

Lines changed: 12 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,23 +12,34 @@
1212
import org.elasticsearch.inference.TaskType;
1313

1414
import java.io.IOException;
15+
import java.util.List;
16+
import java.util.Map;
1517

1618
import static org.elasticsearch.xpack.inference.InferenceBaseRestTest.getAllModels;
1719
import static org.elasticsearch.xpack.inference.InferenceBaseRestTest.getModels;
1820
import static org.hamcrest.Matchers.hasSize;
21+
import static org.hamcrest.Matchers.is;
1922

2023
public class InferenceGetModelsWithElasticInferenceServiceIT extends BaseMockEISAuthServerTest {
2124

2225
public void testGetDefaultEndpoints() throws IOException {
2326
var allModels = getAllModels();
2427
var chatCompletionModels = getModels("_all", TaskType.CHAT_COMPLETION);
2528

26-
assertThat(allModels, hasSize(4));
29+
assertThat(allModels, hasSize(5));
2730
assertThat(chatCompletionModels, hasSize(1));
2831

2932
for (var model : chatCompletionModels) {
3033
assertEquals("chat_completion", model.get("task_type"));
3134
}
3235

36+
assertInferenceIdTaskType(allModels, ".rainbow-sprinkles-elastic", TaskType.CHAT_COMPLETION);
37+
assertInferenceIdTaskType(allModels, ".elser-v2-elastic", TaskType.SPARSE_EMBEDDING);
38+
}
39+
40+
private static void assertInferenceIdTaskType(List<Map<String, Object>> models, String inferenceId, TaskType taskType) {
41+
var model = models.stream().filter(m -> m.get("inference_id").equals(inferenceId)).findFirst();
42+
assertTrue("could not find inference id: " + inferenceId, model.isPresent());
43+
assertThat(model.get().get("task_type"), is(taskType.toString()));
3344
}
3445
}

‎x-pack/plugin/inference/src/internalClusterTest/java/org/elasticsearch/xpack/inference/integration/InferenceRevokeDefaultEndpointsIT.java‎

Lines changed: 11 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -204,6 +204,7 @@ public void testRemoves_DefaultChatCompletion_V1_WhenAuthorizationDoesNotReturnA
204204
service.defaultConfigIds(),
205205
is(
206206
List.of(
207+
new InferenceService.DefaultConfigId(".elser-v2-elastic", MinimalServiceSettings.sparseEmbedding(), service),
207208
new InferenceService.DefaultConfigId(
208209
".rainbow-sprinkles-elastic",
209210
MinimalServiceSettings.chatCompletion(),
@@ -216,7 +217,8 @@ public void testRemoves_DefaultChatCompletion_V1_WhenAuthorizationDoesNotReturnA
216217

217218
PlainActionFuture<List<Model>> listener = new PlainActionFuture<>();
218219
service.defaultConfigs(listener);
219-
assertThat(listener.actionGet(TIMEOUT).get(0).getConfigurations().getInferenceEntityId(), is(".rainbow-sprinkles-elastic"));
220+
assertThat(listener.actionGet(TIMEOUT).get(0).getConfigurations().getInferenceEntityId(), is(".elser-v2-elastic"));
221+
assertThat(listener.actionGet(TIMEOUT).get(1).getConfigurations().getInferenceEntityId(), is(".rainbow-sprinkles-elastic"));
220222

221223
var getModelListener = new PlainActionFuture<UnparsedModel>();
222224
// persists the default endpoints
@@ -244,12 +246,18 @@ public void testRemoves_DefaultChatCompletion_V1_WhenAuthorizationDoesNotReturnA
244246
try (var service = createElasticInferenceService()) {
245247
service.waitForAuthorizationToComplete(TIMEOUT);
246248
assertThat(service.supportedStreamingTasks(), is(EnumSet.noneOf(TaskType.class)));
247-
assertTrue(service.defaultConfigIds().isEmpty());
249+
assertThat(
250+
service.defaultConfigIds(),
251+
is(
252+
List.of(
253+
new InferenceService.DefaultConfigId(".elser-v2-elastic", MinimalServiceSettings.sparseEmbedding(), service)
254+
)
255+
)
256+
);
248257
assertThat(service.supportedTaskTypes(), is(EnumSet.of(TaskType.SPARSE_EMBEDDING)));
249258

250259
var getModelListener = new PlainActionFuture<UnparsedModel>();
251260
modelRegistry.getModel(".rainbow-sprinkles-elastic", getModelListener);
252-
253261
var exception = expectThrows(ResourceNotFoundException.class, () -> getModelListener.actionGet(TIMEOUT));
254262
assertThat(exception.getMessage(), is("Inference endpoint not found [.rainbow-sprinkles-elastic]"));
255263
}

‎x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/registry/ModelRegistry.java‎

Lines changed: 11 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -126,11 +126,20 @@ public boolean containsDefaultConfigId(String inferenceEntityId) {
126126
return defaultConfigIds.containsKey(inferenceEntityId);
127127
}
128128

129+
/**
130+
* Adds the default configuration information if it does not already exist internally.
131+
* @param defaultConfigId the default endpoint information
132+
*/
133+
public synchronized void putDefaultIdIfAbsent(InferenceService.DefaultConfigId defaultConfigId) {
134+
defaultConfigIds.putIfAbsent(defaultConfigId.inferenceId(), defaultConfigId);
135+
}
136+
129137
/**
130138
* Set the default inference ids provided by the services
131-
* @param defaultConfigId The default
139+
* @param defaultConfigId The default endpoint information
140+
* @throws IllegalStateException if the {@link InferenceService.DefaultConfigId#inferenceId()} already exists internally
132141
*/
133-
public synchronized void addDefaultIds(InferenceService.DefaultConfigId defaultConfigId) {
142+
public synchronized void addDefaultIds(InferenceService.DefaultConfigId defaultConfigId) throws IllegalStateException {
134143
var config = defaultConfigIds.get(defaultConfigId.inferenceId());
135144
if (config != null) {
136145
throw new IllegalStateException(

0 commit comments

Comments
 (0)