Skip to content

Commit 4c43deb

Browse files
mjmbischoffmjmbischoff
authored
Merge branch 'main' into esql-string-contains
2 parents 3afaf81 + 5e43ea5 commit 4c43deb

File tree

180 files changed

+2724
-853
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

180 files changed

+2724
-853
lines changed

‎benchmarks/src/main/java/org/elasticsearch/benchmark/_nightly/esql/QueryPlanningBenchmark.java‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -92,7 +92,8 @@ public void setup() {
9292
var fields = 10_000;
9393
var mapping = LinkedHashMap.<String, EsField>newLinkedHashMap(fields);
9494
for (int i = 0; i < fields; i++) {
95-
mapping.put("field" + i, new EsField("field-" + i, TEXT, emptyMap(), true));
95+
// We're creating a standard index, so none of these fields should be marked as dimensions.
96+
mapping.put("field" + i, new EsField("field-" + i, TEXT, emptyMap(), true, EsField.TimeSeriesFieldType.NONE));
9697
}
9798

9899
var esIndex = new EsIndex("test", mapping, Map.of("test", IndexMode.STANDARD));

‎benchmarks/src/main/java/org/elasticsearch/benchmark/compute/operator/EvalBenchmark.java‎

Lines changed: 21 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -212,7 +212,7 @@ private static EvalOperator.ExpressionEvaluator evaluator(String operation) {
212212
FieldAttribute timestamp = new FieldAttribute(
213213
Source.EMPTY,
214214
"timestamp",
215-
new EsField("timestamp", DataType.DATETIME, Map.of(), true)
215+
new EsField("timestamp", DataType.DATETIME, Map.of(), true, EsField.TimeSeriesFieldType.NONE)
216216
);
217217
yield EvalMapper.toEvaluator(
218218
FOLD_CONTEXT,
@@ -321,19 +321,35 @@ private static EvalOperator.ExpressionEvaluator evaluator(String operation) {
321321
}
322322

323323
private static FieldAttribute longField() {
324-
return new FieldAttribute(Source.EMPTY, "long", new EsField("long", DataType.LONG, Map.of(), true));
324+
return new FieldAttribute(
325+
Source.EMPTY,
326+
"long",
327+
new EsField("long", DataType.LONG, Map.of(), true, EsField.TimeSeriesFieldType.NONE)
328+
);
325329
}
326330

327331
private static FieldAttribute doubleField() {
328-
return new FieldAttribute(Source.EMPTY, "double", new EsField("double", DataType.DOUBLE, Map.of(), true));
332+
return new FieldAttribute(
333+
Source.EMPTY,
334+
"double",
335+
new EsField("double", DataType.DOUBLE, Map.of(), true, EsField.TimeSeriesFieldType.NONE)
336+
);
329337
}
330338

331339
private static FieldAttribute intField() {
332-
return new FieldAttribute(Source.EMPTY, "int", new EsField("int", DataType.INTEGER, Map.of(), true));
340+
return new FieldAttribute(
341+
Source.EMPTY,
342+
"int",
343+
new EsField("int", DataType.INTEGER, Map.of(), true, EsField.TimeSeriesFieldType.NONE)
344+
);
333345
}
334346

335347
private static FieldAttribute keywordField() {
336-
return new FieldAttribute(Source.EMPTY, "keyword", new EsField("keyword", DataType.KEYWORD, Map.of(), true));
348+
return new FieldAttribute(
349+
Source.EMPTY,
350+
"keyword",
351+
new EsField("keyword", DataType.KEYWORD, Map.of(), true, EsField.TimeSeriesFieldType.NONE)
352+
);
337353
}
338354

339355
private static Configuration configuration() {

‎build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/transport/AbstractTransportVersionFuncTest.groovy‎

Lines changed: 122 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,122 @@
1+
/*
2+
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
3+
* or more contributor license agreements. Licensed under the "Elastic License
4+
* 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
5+
* Public License v 1"; you may not use this file except in compliance with, at
6+
* your election, the "Elastic License 2.0", the "GNU Affero General Public
7+
* License v3.0 only", or the "Server Side Public License, v 1".
8+
*/
9+
10+
package org.elasticsearch.gradle.internal.transport
11+
12+
import org.elasticsearch.gradle.fixtures.AbstractGradleFuncTest
13+
import org.gradle.testkit.runner.BuildResult
14+
import org.gradle.testkit.runner.TaskOutcome
15+
16+
class AbstractTransportVersionFuncTest extends AbstractGradleFuncTest {
17+
def javaResource(String project, String path, String content) {
18+
file("${project}/src/main/resources/${path}").withWriter { writer ->
19+
writer << content
20+
}
21+
}
22+
23+
def javaSource(String project, String packageName, String className, String imports, String content) {
24+
String packageSlashes = packageName.replace('.', '/')
25+
file("${project}/src/main/java/${packageSlashes}/${className}.java").withWriter { writer ->
26+
writer << """
27+
package ${packageName};
28+
${imports}
29+
public class ${className} {
30+
${content}
31+
}
32+
"""
33+
}
34+
}
35+
36+
def namedTransportVersion(String name, String ids) {
37+
javaResource("myserver", "transport/definitions/named/" + name + ".csv", ids)
38+
}
39+
40+
def unreferencedTransportVersion(String name, String id) {
41+
javaResource("myserver", "transport/definitions/unreferenced/" + name + ".csv", id)
42+
}
43+
44+
def definedAndUsedTransportVersion(String name, String ids) {
45+
return definedAndUsedTransportVersion(name, ids, "Test${name.capitalize()}")
46+
}
47+
48+
def definedAndUsedTransportVersion(String name, String ids, String classname) {
49+
javaSource("myserver", "org.elasticsearch", classname, "", """
50+
static final TransportVersion usage = TransportVersion.fromName("${name}");
51+
""")
52+
namedTransportVersion(name, ids)
53+
}
54+
55+
def latestTransportVersion(String branch, String name, String id) {
56+
javaResource("myserver", "transport/latest/" + branch + ".csv","${name},${id}")
57+
}
58+
59+
def validateReferencesFails(String project) {
60+
return gradleRunner(":${project}:validateTransportVersionReferences").buildAndFail()
61+
}
62+
63+
def validateDefinitionsFails() {
64+
return gradleRunner(":myserver:validateTransportVersionDefinitions").buildAndFail()
65+
}
66+
67+
def assertReferencesFailure(BuildResult result, String project, String expectedOutput) {
68+
result.task(":${project}:validateTransportVersionReferences").outcome == TaskOutcome.FAILED
69+
assertOutputContains(result.output, expectedOutput)
70+
}
71+
72+
def assertDefinitionsFailure(BuildResult result, String expectedOutput) {
73+
result.task(":myserver:validateTransportVersionDefinitions").outcome == TaskOutcome.FAILED
74+
assertOutputContains(result.output, expectedOutput)
75+
}
76+
77+
def setup() {
78+
configurationCacheCompatible = false
79+
internalBuild()
80+
settingsFile << """
81+
include ':myserver'
82+
include ':myplugin'
83+
"""
84+
file("gradle.properties") << """
85+
org.elasticsearch.transport.definitionsProject=:myserver
86+
"""
87+
88+
file("myserver/build.gradle") << """
89+
apply plugin: 'java-library'
90+
apply plugin: 'elasticsearch.transport-version-references'
91+
apply plugin: 'elasticsearch.transport-version-resources'
92+
"""
93+
namedTransportVersion("existing_91", "8012000")
94+
namedTransportVersion("existing_92", "8123000,8012001")
95+
unreferencedTransportVersion("initial_9_0_0", "8000000")
96+
latestTransportVersion("9.2", "existing_92", "8123000")
97+
latestTransportVersion("9.1", "existing_92", "8012001")
98+
// a mock version of TransportVersion, just here so we can compile Dummy.java et al
99+
javaSource("myserver", "org.elasticsearch", "TransportVersion", "", """
100+
public static TransportVersion fromName(String name) {
101+
return null;
102+
}
103+
""")
104+
javaSource("myserver", "org.elasticsearch", "Dummy", "", """
105+
static final TransportVersion existing91 = TransportVersion.fromName("existing_91");
106+
static final TransportVersion existing92 = TransportVersion.fromName("existing_92");
107+
""")
108+
109+
file("myplugin/build.gradle") << """
110+
apply plugin: 'java-library'
111+
apply plugin: 'elasticsearch.transport-version-references'
112+
113+
dependencies {
114+
implementation project(":myserver")
115+
}
116+
"""
117+
118+
setupLocalGitRepo()
119+
execute("git checkout -b main")
120+
execute("git checkout -b test")
121+
}
122+
}

‎build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/transport/TransportVersionManagementPluginFuncTest.groovy‎ renamed to ‎build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/transport/TransportVersionValidationFuncTest.groovy‎

Lines changed: 11 additions & 116 deletions
Original file line numberDiff line numberDiff line change
@@ -10,124 +10,9 @@
1010
package org.elasticsearch.gradle.internal.transport
1111

1212

13-
import org.elasticsearch.gradle.fixtures.AbstractGradleFuncTest
14-
import org.gradle.testkit.runner.BuildResult
1513
import org.gradle.testkit.runner.TaskOutcome
1614

17-
class TransportVersionManagementPluginFuncTest extends AbstractGradleFuncTest {
18-
19-
/**
20-
*
21-
* @param project
22-
* @param path
23-
* @param content
24-
* @return
25-
*/
26-
def javaResource(String project, String path, String content) {
27-
file("${project}/src/main/resources/${path}").withWriter { writer ->
28-
writer << content
29-
}
30-
}
31-
32-
def javaSource(String project, String packageName, String className, String imports, String content) {
33-
String packageSlashes = packageName.replace('.', '/')
34-
file("${project}/src/main/java/${packageSlashes}/${className}.java").withWriter { writer ->
35-
writer << """
36-
package ${packageName};
37-
${imports}
38-
public class ${className} {
39-
${content}
40-
}
41-
"""
42-
}
43-
}
44-
45-
def namedTransportVersion(String name, String ids) {
46-
javaResource("myserver", "transport/definitions/named/" + name + ".csv", ids)
47-
}
48-
49-
def initialTransportVersion(String name, String id) {
50-
javaResource("myserver", "transport/definitions/initial/" + name + ".csv", id)
51-
}
52-
53-
def definedAndUsedTransportVersion(String name, String ids) {
54-
return definedAndUsedTransportVersion(name, ids, "Test${name.capitalize()}")
55-
}
56-
57-
def definedAndUsedTransportVersion(String name, String ids, String classname) {
58-
javaSource("myserver", "org.elasticsearch", classname, "", """
59-
static final TransportVersion usage = TransportVersion.fromName("${name}");
60-
""")
61-
namedTransportVersion(name, ids)
62-
}
63-
64-
def latestTransportVersion(String branch, String name, String id) {
65-
javaResource("myserver", "transport/latest/" + branch + ".csv","${name},${id}")
66-
}
67-
68-
def validateReferencesFails(String project) {
69-
return gradleRunner(":${project}:validateTransportVersionReferences").buildAndFail()
70-
}
71-
72-
def validateDefinitionsFails() {
73-
return gradleRunner(":myserver:validateTransportVersionDefinitions").buildAndFail()
74-
}
75-
76-
def assertReferencesFailure(BuildResult result, String project, String expectedOutput) {
77-
result.task(":${project}:validateTransportVersionReferences").outcome == TaskOutcome.FAILED
78-
assertOutputContains(result.output, expectedOutput)
79-
}
80-
81-
def assertDefinitionsFailure(BuildResult result, String expectedOutput) {
82-
result.task(":myserver:validateTransportVersionDefinitions").outcome == TaskOutcome.FAILED
83-
assertOutputContains(result.output, expectedOutput)
84-
}
85-
86-
def setup() {
87-
configurationCacheCompatible = false
88-
internalBuild()
89-
settingsFile << """
90-
include ':myserver'
91-
include ':myplugin'
92-
"""
93-
file("gradle.properties") << """
94-
org.elasticsearch.transport.definitionsProject=:myserver
95-
"""
96-
97-
file("myserver/build.gradle") << """
98-
apply plugin: 'java-library'
99-
apply plugin: 'elasticsearch.transport-version-references'
100-
apply plugin: 'elasticsearch.transport-version-resources'
101-
"""
102-
namedTransportVersion("existing_91", "8012000")
103-
namedTransportVersion("existing_92", "8123000,8012001")
104-
initialTransportVersion("initial_9_0_0", "8000000")
105-
latestTransportVersion("9.2", "existing_92", "8123000")
106-
latestTransportVersion("9.1", "existing_92", "8012001")
107-
// a mock version of TransportVersion, just here so we can compile Dummy.java et al
108-
javaSource("myserver", "org.elasticsearch", "TransportVersion", "", """
109-
public static TransportVersion fromName(String name) {
110-
return null;
111-
}
112-
""")
113-
javaSource("myserver", "org.elasticsearch", "Dummy", "", """
114-
static final TransportVersion existing91 = TransportVersion.fromName("existing_91");
115-
static final TransportVersion existing92 = TransportVersion.fromName("existing_92");
116-
""")
117-
118-
file("myplugin/build.gradle") << """
119-
apply plugin: 'java-library'
120-
apply plugin: 'elasticsearch.transport-version-references'
121-
122-
dependencies {
123-
implementation project(":myserver")
124-
}
125-
"""
126-
127-
setupLocalGitRepo()
128-
execute("git checkout -b main")
129-
execute("git checkout -b test")
130-
}
15+
class TransportVersionValidationFuncTest extends AbstractTransportVersionFuncTest {
13116

13217
def "test setup works"() {
13318
when:
@@ -303,4 +188,14 @@ class TransportVersionManagementPluginFuncTest extends AbstractGradleFuncTest {
303188
assertDefinitionsFailure(result, "Transport version definition file " +
304189
"[myserver/src/main/resources/transport/definitions/named/patch.csv] has patch version 8015001 as primary id")
305190
}
191+
192+
def "unreferenced directory is optional"() {
193+
given:
194+
file("myserver/src/main/resources/transport/unreferenced/initial_9_0_0.csv").delete()
195+
file("myserver/src/main/resources/transport/unreferenced").deleteDir()
196+
when:
197+
def result = gradleRunner(":myserver:validateTransportVersionDefinitions").build()
198+
then:
199+
result.task(":myserver:validateTransportVersionDefinitions").outcome == TaskOutcome.SUCCESS
200+
}
306201
}

‎build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/ValidateTransportVersionResourcesTask.java‎

Lines changed: 7 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -107,10 +107,13 @@ public void validateTransportVersions() throws IOException {
107107
// now load all definitions, do some validation and record them by various keys for later quick lookup
108108
// NOTE: this must run after loading referenced names and existing definitions
109109
// NOTE: this is sorted so that the order of cross validation is deterministic
110-
for (String subDir : List.of("initial", "named")) {
111-
try (var definitionsStream = Files.list(definitionsDir.resolve(subDir)).sorted()) {
112-
for (var definitionFile : definitionsStream.toList()) {
113-
recordAndValidateDefinition(readDefinitionFile(definitionFile));
110+
for (String subDirName : List.of("unreferenced", "named")) {
111+
Path subDir = definitionsDir.resolve(subDirName);
112+
if (Files.isDirectory(subDir)) {
113+
try (var definitionsStream = Files.list(subDir).sorted()) {
114+
for (var definitionFile : definitionsStream.toList()) {
115+
recordAndValidateDefinition(readDefinitionFile(definitionFile));
116+
}
114117
}
115118
}
116119
}

‎catalog-info.yaml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -306,7 +306,7 @@ spec:
306306
apiVersion: backstage.io/v1alpha1
307307
kind: Resource
308308
metadata:
309-
name: buildkite-pipeline-elasticsearch-pull-request-performance-benchmark
309+
name: buildkite-pipeline-elasticsearch-pull-request-perf-benchmark
310310
description: Elasticsearch pull request performance benchmark
311311
links:
312312
- title: Pipeline

‎docs/changelog/131907.yaml‎

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,26 @@
1+
pr: 131907
2+
summary: Enable `exclude_source_vectors` by default for new indices
3+
area: Vector Search
4+
type: breaking
5+
issues: []
6+
breaking:
7+
title: Enable `exclude_source_vectors` by default for new indices
8+
area: Search
9+
details: |-
10+
The `exclude_source_vectors` setting is now enabled by default for newly created indices.
11+
This means that vector fields (e.g., `dense_vector`) are no longer stored in the `_source` field
12+
by default, although they remain fully accessible through search and retrieval operations.
13+
14+
Instead of being persisted in `_source`, vectors are now rehydrated on demand from the underlying
15+
index structures when needed. This reduces index size and improves performance for typical vector
16+
search workloads where the original vector values do not need to be part of the `_source`.
17+
18+
If your use case requires vector fields to be stored in `_source`, you can disable this behavior by
19+
setting `exclude_source_vectors: false` at index creation time.
20+
impact: |-
21+
Vector fields will no longer be stored in `_source` by default for new indices. Applications or tools
22+
that expect to see vector fields in `_source` (for raw document inspection)
23+
may need to be updated or configured to explicitly retain vectors using `exclude_source_vectors: false`.
24+
25+
Retrieval of vector fields via search or the `_source` API remains fully supported.
26+
notable: true

‎docs/changelog/132766.yaml‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
pr: 132766
2+
summary: Change `reporting_user` role to leverage reserved kibana privileges
3+
area: Authorization
4+
type: deprecation
5+
issues: []
6+
deprecation:
7+
title: Deprecate the built-in `reporting_user` role.
8+
area: Authorization
9+
details: The `reporting_user` role is deprecated. Administrators should manage access to Kibana's reporting features via custom roles which grant the necessary privileges.
10+
impact: This role will be removed in a future version. Administrators should migrate to custom roles to avoid interruption.

‎docs/changelog/132845.yaml‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
pr: 132845
2+
summary: Expose existing DLS cache x-pack usage statistics
3+
area: Authorization
4+
type: enhancement
5+
issues: []

‎docs/changelog/132858.yaml‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
pr: 132858
2+
summary: Add index mode to resolve index response
3+
area: Indices APIs
4+
type: feature
5+
issues: []

0 commit comments

Comments
 (0)