Skip to content

Commit ecd2a0f

Browse files
samxbrsamxbr
authored
Merge branch 'main' into feature/slm/shutdown-aware
2 parents 44effa1 + 47e0524 commit ecd2a0f

File tree

41 files changed

+1775
-373
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

41 files changed

+1775
-373
lines changed

‎benchmarks/src/main/java/org/elasticsearch/benchmark/compute/operator/ValuesAggregatorBenchmark.java‎

Lines changed: 20 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -113,16 +113,16 @@ static void selfTest() {
113113
@Param({ BYTES_REF, INT, LONG })
114114
public String dataType;
115115

116-
private static Operator operator(DriverContext driverContext, int groups, String dataType) {
116+
private static Operator operator(DriverContext driverContext, int groups, String dataType, AggregatorMode mode) {
117117
if (groups == 1) {
118118
return new AggregationOperator(
119-
List.of(supplier(dataType).aggregatorFactory(AggregatorMode.SINGLE, List.of(0)).apply(driverContext)),
119+
List.of(supplier(dataType).aggregatorFactory(mode, List.of(0)).apply(driverContext)),
120120
driverContext
121121
);
122122
}
123123
List<BlockHash.GroupSpec> groupSpec = List.of(new BlockHash.GroupSpec(0, ElementType.LONG));
124124
return new HashAggregationOperator(
125-
List.of(supplier(dataType).groupingAggregatorFactory(AggregatorMode.SINGLE, List.of(1))),
125+
List.of(supplier(dataType).groupingAggregatorFactory(mode, List.of(1))),
126126
() -> BlockHash.build(groupSpec, driverContext.blockFactory(), 16 * 1024, false),
127127
driverContext
128128
) {
@@ -177,6 +177,9 @@ private static void checkGrouped(String prefix, int groups, String dataType, Pag
177177

178178
// Check them
179179
BytesRefBlock values = page.getBlock(1);
180+
if (values.asOrdinals() == null) {
181+
throw new AssertionError(" expected ordinals; but got " + values);
182+
}
180183
for (int p = 0; p < groups; p++) {
181184
checkExpectedBytesRef(prefix, values, p, expected.get(p));
182185
}
@@ -341,13 +344,21 @@ public void run() {
341344

342345
private static void run(int groups, String dataType, int opCount) {
343346
DriverContext driverContext = driverContext();
344-
try (Operator operator = operator(driverContext, groups, dataType)) {
345-
Page page = page(groups, dataType);
346-
for (int i = 0; i < opCount; i++) {
347-
operator.addInput(page.shallowCopy());
347+
try (Operator finalAggregator = operator(driverContext, groups, dataType, AggregatorMode.FINAL)) {
348+
try (Operator initialAggregator = operator(driverContext, groups, dataType, AggregatorMode.INITIAL)) {
349+
Page rawPage = page(groups, dataType);
350+
for (int i = 0; i < opCount; i++) {
351+
initialAggregator.addInput(rawPage.shallowCopy());
352+
}
353+
initialAggregator.finish();
354+
Page intermediatePage = initialAggregator.getOutput();
355+
for (int i = 0; i < opCount; i++) {
356+
finalAggregator.addInput(intermediatePage.shallowCopy());
357+
}
348358
}
349-
operator.finish();
350-
checkExpected(groups, dataType, operator.getOutput());
359+
finalAggregator.finish();
360+
Page outputPage = finalAggregator.getOutput();
361+
checkExpected(groups, dataType, outputPage);
351362
}
352363
}
353364

‎build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/toolchain/OracleOpenJdkToolchainResolver.java‎

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -61,11 +61,17 @@ public String url(String os, String arch, String extension) {
6161
record EarlyAccessJdkBuild(JavaLanguageVersion languageVersion, String buildNumber) implements JdkBuild {
6262
@Override
6363
public String url(String os, String arch, String extension) {
64-
return "https://download.java.net/java/early_access/jdk"
64+
// example:
65+
// http://builds.es-jdk-archive.com/jdks/openjdk/26/openjdk-26-ea+6/openjdk-26-ea+6_linux-aarch64_bin.tar.gz
66+
return "http://builds.es-jdk-archive.com/jdks/openjdk/"
6567
+ languageVersion.asInt()
6668
+ "/"
69+
+ "openjdk-"
70+
+ languageVersion.asInt()
71+
+ "-ea+"
6772
+ buildNumber
68-
+ "/GPL/openjdk-"
73+
+ "/"
74+
+ "openjdk-"
6975
+ languageVersion.asInt()
7076
+ "-ea+"
7177
+ buildNumber

‎build-tools-internal/src/test/groovy/org/elasticsearch/gradle/internal/toolchain/OracleOpenJdkToolchainResolverSpec.groovy‎

Lines changed: 21 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -67,16 +67,16 @@ class OracleOpenJdkToolchainResolverSpec extends AbstractToolchainResolverSpec {
6767
[24, anyVendor(), LINUX, AARCH64, "https://download.oracle.com/java/GA/jdk24/1f9ff9062db4449d8ca828c504ffae90/36/GPL/openjdk-24_linux-aarch64_bin.tar.gz"],
6868
[24, anyVendor(), WINDOWS, X86_64, "https://download.oracle.com/java/GA/jdk24/1f9ff9062db4449d8ca828c504ffae90/36/GPL/openjdk-24_windows-x64_bin.zip"],
6969
// EA build
70-
[25, ORACLE, MAC_OS, X86_64, "https://download.java.net/java/early_access/jdk25/3/GPL/openjdk-25-ea+3_macos-x64_bin.tar.gz"],
71-
[25, ORACLE, MAC_OS, AARCH64, "https://download.java.net/java/early_access/jdk25/3/GPL/openjdk-25-ea+3_macos-aarch64_bin.tar.gz"],
72-
[25, ORACLE, LINUX, X86_64, "https://download.java.net/java/early_access/jdk25/3/GPL/openjdk-25-ea+3_linux-x64_bin.tar.gz"],
73-
[25, ORACLE, LINUX, AARCH64, "https://download.java.net/java/early_access/jdk25/3/GPL/openjdk-25-ea+3_linux-aarch64_bin.tar.gz"],
74-
[25, ORACLE, WINDOWS, X86_64, "https://download.java.net/java/early_access/jdk25/3/GPL/openjdk-25-ea+3_windows-x64_bin.zip"],
75-
[25, anyVendor(), MAC_OS, X86_64, "https://download.java.net/java/early_access/jdk25/3/GPL/openjdk-25-ea+3_macos-x64_bin.tar.gz"],
76-
[25, anyVendor(), MAC_OS, AARCH64, "https://download.java.net/java/early_access/jdk25/3/GPL/openjdk-25-ea+3_macos-aarch64_bin.tar.gz"],
77-
[25, anyVendor(), LINUX, X86_64, "https://download.java.net/java/early_access/jdk25/3/GPL/openjdk-25-ea+3_linux-x64_bin.tar.gz"],
78-
[25, anyVendor(), LINUX, AARCH64, "https://download.java.net/java/early_access/jdk25/3/GPL/openjdk-25-ea+3_linux-aarch64_bin.tar.gz"],
79-
[25, anyVendor(), WINDOWS, X86_64, "https://download.java.net/java/early_access/jdk25/3/GPL/openjdk-25-ea+3_windows-x64_bin.zip"]]
70+
[25, ORACLE, MAC_OS, X86_64, "http://builds.es-jdk-archive.com/jdks/openjdk/25/openjdk-25-ea+3/openjdk-25-ea+3_macos-x64_bin.tar.gz"],
71+
[25, ORACLE, MAC_OS, AARCH64, "http://builds.es-jdk-archive.com/jdks/openjdk/25/openjdk-25-ea+3/openjdk-25-ea+3_macos-aarch64_bin.tar.gz"],
72+
[25, ORACLE, LINUX, X86_64, "http://builds.es-jdk-archive.com/jdks/openjdk/25/openjdk-25-ea+3/openjdk-25-ea+3_linux-x64_bin.tar.gz"],
73+
[25, ORACLE, LINUX, AARCH64, "http://builds.es-jdk-archive.com/jdks/openjdk/25/openjdk-25-ea+3/openjdk-25-ea+3_linux-aarch64_bin.tar.gz"],
74+
[25, ORACLE, WINDOWS, X86_64, "http://builds.es-jdk-archive.com/jdks/openjdk/25/openjdk-25-ea+3/openjdk-25-ea+3_windows-x64_bin.zip"],
75+
[25, anyVendor(), MAC_OS, X86_64, "http://builds.es-jdk-archive.com/jdks/openjdk/25/openjdk-25-ea+3/openjdk-25-ea+3_macos-x64_bin.tar.gz"],
76+
[25, anyVendor(), MAC_OS, AARCH64, "http://builds.es-jdk-archive.com/jdks/openjdk/25/openjdk-25-ea+3/openjdk-25-ea+3_macos-aarch64_bin.tar.gz"],
77+
[25, anyVendor(), LINUX, X86_64, "http://builds.es-jdk-archive.com/jdks/openjdk/25/openjdk-25-ea+3/openjdk-25-ea+3_linux-x64_bin.tar.gz"],
78+
[25, anyVendor(), LINUX, AARCH64, "http://builds.es-jdk-archive.com/jdks/openjdk/25/openjdk-25-ea+3/openjdk-25-ea+3_linux-aarch64_bin.tar.gz"],
79+
[25, anyVendor(), WINDOWS, X86_64, "http://builds.es-jdk-archive.com/jdks/openjdk/25/openjdk-25-ea+3/openjdk-25-ea+3_windows-x64_bin.zip"]]
8080
}
8181

8282
@RestoreSystemProperties
@@ -100,20 +100,20 @@ class OracleOpenJdkToolchainResolverSpec extends AbstractToolchainResolverSpec {
100100

101101
where:
102102
version | vendor | os | arch | expectedUrl
103-
25 | ORACLE | MAC_OS | X86_64 | urlPrefix(25) + "13/GPL/openjdk-25-ea+13_macos-x64_bin.tar.gz"
104-
25 | ORACLE | MAC_OS | AARCH64 | urlPrefix(25) + "13/GPL/openjdk-25-ea+13_macos-aarch64_bin.tar.gz"
105-
25 | ORACLE | LINUX | X86_64 | urlPrefix(25) + "13/GPL/openjdk-25-ea+13_linux-x64_bin.tar.gz"
106-
25 | ORACLE | LINUX | AARCH64 | urlPrefix(25) + "13/GPL/openjdk-25-ea+13_linux-aarch64_bin.tar.gz"
107-
25 | ORACLE | WINDOWS | X86_64 | urlPrefix(25) + "13/GPL/openjdk-25-ea+13_windows-x64_bin.zip"
108-
25 | anyVendor() | MAC_OS | X86_64 | urlPrefix(25) + "13/GPL/openjdk-25-ea+13_macos-x64_bin.tar.gz"
109-
25 | anyVendor() | MAC_OS | AARCH64 | urlPrefix(25) + "13/GPL/openjdk-25-ea+13_macos-aarch64_bin.tar.gz"
110-
25 | anyVendor() | LINUX | X86_64 | urlPrefix(25) + "13/GPL/openjdk-25-ea+13_linux-x64_bin.tar.gz"
111-
25 | anyVendor() | LINUX | AARCH64 | urlPrefix(25) + "13/GPL/openjdk-25-ea+13_linux-aarch64_bin.tar.gz"
112-
25 | anyVendor() | WINDOWS | X86_64 | urlPrefix(25) + "13/GPL/openjdk-25-ea+13_windows-x64_bin.zip"
103+
25 | ORACLE | MAC_OS | X86_64 | urlPrefix(25) + "openjdk-25-ea+13/openjdk-25-ea+13_macos-x64_bin.tar.gz"
104+
25 | ORACLE | MAC_OS | AARCH64 | urlPrefix(25) + "openjdk-25-ea+13/openjdk-25-ea+13_macos-aarch64_bin.tar.gz"
105+
25 | ORACLE | LINUX | X86_64 | urlPrefix(25) + "openjdk-25-ea+13/openjdk-25-ea+13_linux-x64_bin.tar.gz"
106+
25 | ORACLE | LINUX | AARCH64 | urlPrefix(25) + "openjdk-25-ea+13/openjdk-25-ea+13_linux-aarch64_bin.tar.gz"
107+
25 | ORACLE | WINDOWS | X86_64 | urlPrefix(25) + "openjdk-25-ea+13/openjdk-25-ea+13_windows-x64_bin.zip"
108+
25 | anyVendor() | MAC_OS | X86_64 | urlPrefix(25) + "openjdk-25-ea+13/openjdk-25-ea+13_macos-x64_bin.tar.gz"
109+
25 | anyVendor() | MAC_OS | AARCH64 | urlPrefix(25) + "openjdk-25-ea+13/openjdk-25-ea+13_macos-aarch64_bin.tar.gz"
110+
25 | anyVendor() | LINUX | X86_64 | urlPrefix(25) + "openjdk-25-ea+13/openjdk-25-ea+13_linux-x64_bin.tar.gz"
111+
25 | anyVendor() | LINUX | AARCH64 | urlPrefix(25) + "openjdk-25-ea+13/openjdk-25-ea+13_linux-aarch64_bin.tar.gz"
112+
25 | anyVendor() | WINDOWS | X86_64 | urlPrefix(25) + "openjdk-25-ea+13/openjdk-25-ea+13_windows-x64_bin.zip"
113113
}
114114

115115
private static String urlPrefix(int i) {
116-
return "https://download.java.net/java/early_access/jdk" + i + "/"
116+
return "http://builds.es-jdk-archive.com/jdks/openjdk/" + i + "/"
117117
}
118118

119119
def unsupportedRequests() {

‎docs/changelog/130463.yaml‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
pr: 130463
2+
summary: Refresh potential lost connections at query start for `_search`
3+
area: Search
4+
type: enhancement
5+
issues: []

‎docs/changelog/131236.yaml‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
pr: 131236
2+
summary: Correctly handling `download_database_on_pipeline_creation` within a pipeline
3+
processor within a default or final pipeline
4+
area: Ingest Node
5+
type: bug
6+
issues: []

‎docs/changelog/131390.yaml‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
pr: 131390
2+
summary: Add optimized path for intermediate values aggregator
3+
area: ES|QL
4+
type: enhancement
5+
issues: []

‎docs/changelog/131531.yaml‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
pr: 131531
2+
summary: Integrate LIKE/RLIKE LIST with `ReplaceStringCasingWithInsensitiveRegexMatch`
3+
rule
4+
area: ES|QL
5+
type: enhancement
6+
issues: []

‎docs/reference/query-languages/esql/_snippets/commands/layout/completion.md‎

Lines changed: 106 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,106 @@
1+
## `COMPLETION` [esql-completion]
2+
3+
```yaml {applies_to}
4+
serverless: preview
5+
stack: preview 9.1.0
6+
```
7+
8+
The `COMPLETION` command allows you to send prompts and context to a Large Language Model (LLM) directly within your ES|QL queries, to perform text generation tasks.
9+
10+
**Syntax**
11+
12+
```esql
13+
COMPLETION [column =] prompt WITH inference_id
14+
```
15+
16+
**Parameters**
17+
18+
`column`
19+
: (Optional) The name of the output column containing the LLM's response.
20+
If not specified, the results will be stored in a column named `completion`.
21+
If the specified column already exists, it will be overwritten with the new results.
22+
23+
`prompt`
24+
: The input text or expression used to prompt the LLM.
25+
This can be a string literal or a reference to a column containing text.
26+
27+
`inference_id`
28+
: The ID of the [inference endpoint](docs-content://explore-analyze/elastic-inference/inference-api.md) to use for the task.
29+
The inference endpoint must be configured with the `completion` task type.
30+
31+
**Description**
32+
33+
The `COMPLETION` command provides a general-purpose interface for
34+
text generation tasks using a Large Language Model (LLM) in ES|QL.
35+
36+
`COMPLETION` supports a wide range of text generation tasks. Depending on your
37+
prompt and the model you use, you can perform arbitrary text generation tasks
38+
including:
39+
40+
- Question answering
41+
- Summarization
42+
- Translation
43+
- Content rewriting
44+
- Creative generation
45+
46+
**Requirements**
47+
48+
To use this command, you must deploy your LLM model in Elasticsearch as
49+
an [≈inference endpoint](https://www.elastic.co/docs/api/doc/elasticsearch/operation/operation-inference-put) with the
50+
task type `completion`.
51+
52+
**Examples**
53+
54+
Use the default column name (results stored in `completion` column):
55+
56+
```esql
57+
ROW question = "What is Elasticsearch?"
58+
| COMPLETION question WITH test_completion_model
59+
| KEEP question, completion
60+
```
61+
62+
| question:keyword | completion:keyword |
63+
|------------------------|-------------------------------------------|
64+
| What is Elasticsearch? | A distributed search and analytics engine |
65+
66+
Specify the output column (results stored in `answer` column):
67+
68+
```esql
69+
ROW question = "What is Elasticsearch?"
70+
| COMPLETION answer = question WITH test_completion_model
71+
| KEEP question, answer
72+
```
73+
74+
| question:keyword | answer:keyword |
75+
| --- | --- |
76+
| What is Elasticsearch? | A distributed search and analytics engine |
77+
78+
Summarize the top 10 highest-rated movies using a prompt:
79+
80+
```esql
81+
FROM movies
82+
| SORT rating DESC
83+
| LIMIT 10
84+
| EVAL prompt = CONCAT(
85+
"Summarize this movie using the following information: \n",
86+
"Title: ", title, "\n",
87+
"Synopsis: ", synopsis, "\n",
88+
"Actors: ", MV_CONCAT(actors, ", "), "\n",
89+
)
90+
| COMPLETION summary = prompt WITH test_completion_model
91+
| KEEP title, summary, rating
92+
```
93+
94+
95+
| title:keyword | summary:keyword | rating:double |
96+
| --- | --- | --- |
97+
| The Shawshank Redemption | A tale of hope and redemption in prison. | 9.3 |
98+
| The Godfather | A mafia family's rise and fall. | 9.2 |
99+
| The Dark Knight | Batman battles the Joker in Gotham. | 9.0 |
100+
| Pulp Fiction | Interconnected crime stories with dark humor. | 8.9 |
101+
| Fight Club | A man starts an underground fight club. | 8.8 |
102+
| Inception | A thief steals secrets through dreams. | 8.8 |
103+
| The Matrix | A hacker discovers reality is a simulation. | 8.7 |
104+
| Parasite | Class conflict between two families. | 8.6 |
105+
| Interstellar | A team explores space to save humanity. | 8.6 |
106+
| The Prestige | Rival magicians engage in dangerous competition. | 8.5 |

‎docs/reference/query-languages/esql/_snippets/lists/processing-commands.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
* [preview] [`CHANGE_POINT`](../../commands/processing-commands.md#esql-change_point)
2+
* [preview] [`COMPLETION`](../../commands/processing-commands.md#esql-completion)
23
* [`DISSECT`](../../commands/processing-commands.md#esql-dissect)
34
* [`DROP`](../../commands/processing-commands.md#esql-drop)
45
* [`ENRICH`](../../commands/processing-commands.md#esql-enrich)

‎docs/reference/query-languages/esql/commands/processing-commands.md‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,9 @@ mapped_pages:
2020
:::{include} ../_snippets/commands/layout/change_point.md
2121
:::
2222

23+
:::{include} ../_snippets/commands/layout/completion.md
24+
:::
25+
2326
:::{include} ../_snippets/commands/layout/dissect.md
2427
:::
2528

0 commit comments

Comments
 (0)