Skip to content

Commit e53093f

Browse files
prwhelanprwhelan
authored
Merge branch 'main' into refactor/streaming-metrics
2 parents b6b13dd + cda2669 commit e53093f

File tree

736 files changed

+14195
-6363
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

736 files changed

+14195
-6363
lines changed

‎build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/InternalDistributionArchiveCheckPlugin.java‎

Lines changed: 37 additions & 37 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,8 @@
1919
import org.gradle.api.Task;
2020
import org.gradle.api.file.ArchiveOperations;
2121
import org.gradle.api.plugins.BasePlugin;
22+
import org.gradle.api.provider.ListProperty;
23+
import org.gradle.api.provider.Provider;
2224
import org.gradle.api.tasks.Copy;
2325
import org.gradle.api.tasks.TaskProvider;
2426

@@ -103,22 +105,26 @@ private static TaskProvider<Task> registerCheckMlCppNoticeTask(
103105
) {
104106
TaskProvider<Task> checkMlCppNoticeTask = project.getTasks().register("checkMlCppNotice", task -> {
105107
task.dependsOn(checkExtraction);
108+
final Provider<Path> noticePath = checkExtraction.map(
109+
c -> c.getDestinationDir()
110+
.toPath()
111+
.resolve("elasticsearch-" + VersionProperties.getElasticsearch() + "/modules/x-pack-ml/NOTICE.txt")
112+
);
113+
ListProperty<String> expectedMlLicenses = extension.expectedMlLicenses;
106114
task.doLast(new Action<Task>() {
107115
@Override
108116
public void execute(Task task) {
109117
// this is just a small sample from the C++ notices,
110118
// the idea being that if we've added these lines we've probably added all the required lines
111-
final List<String> expectedLines = extension.expectedMlLicenses.get();
112-
final Path noticePath = checkExtraction.get()
113-
.getDestinationDir()
114-
.toPath()
115-
.resolve("elasticsearch-" + VersionProperties.getElasticsearch() + "/modules/x-pack-ml/NOTICE.txt");
119+
final List<String> expectedLines = expectedMlLicenses.get();
116120
final List<String> actualLines;
117121
try {
118-
actualLines = Files.readAllLines(noticePath);
122+
actualLines = Files.readAllLines(noticePath.get());
119123
for (final String expectedLine : expectedLines) {
120124
if (actualLines.contains(expectedLine) == false) {
121-
throw new GradleException("expected [" + noticePath + " to contain [" + expectedLine + "] but it did not");
125+
throw new GradleException(
126+
"expected [" + noticePath.get() + " to contain [" + expectedLine + "] but it did not"
127+
);
122128
}
123129
}
124130
} catch (IOException ioException) {
@@ -133,43 +139,37 @@ public void execute(Task task) {
133139
private TaskProvider<Task> registerCheckNoticeTask(Project project, TaskProvider<Copy> checkExtraction) {
134140
return project.getTasks().register("checkNotice", task -> {
135141
task.dependsOn(checkExtraction);
136-
task.doLast(new Action<Task>() {
137-
@Override
138-
public void execute(Task task) {
139-
final List<String> noticeLines = Arrays.asList("Elasticsearch", "Copyright 2009-2024 Elasticsearch");
140-
final Path noticePath = checkExtraction.get()
141-
.getDestinationDir()
142-
.toPath()
143-
.resolve("elasticsearch-" + VersionProperties.getElasticsearch() + "/NOTICE.txt");
144-
assertLinesInFile(noticePath, noticeLines);
145-
}
142+
var noticePath = checkExtraction.map(
143+
copy -> copy.getDestinationDir().toPath().resolve("elasticsearch-" + VersionProperties.getElasticsearch() + "/NOTICE.txt")
144+
);
145+
task.doLast(t -> {
146+
final List<String> noticeLines = Arrays.asList("Elasticsearch", "Copyright 2009-2024 Elasticsearch");
147+
assertLinesInFile(noticePath.get(), noticeLines);
146148
});
147149
});
148150
}
149151

150152
private TaskProvider<Task> registerCheckLicenseTask(Project project, TaskProvider<Copy> checkExtraction) {
151153
TaskProvider<Task> checkLicense = project.getTasks().register("checkLicense", task -> {
152154
task.dependsOn(checkExtraction);
153-
task.doLast(new Action<Task>() {
154-
@Override
155-
public void execute(Task task) {
156-
String licenseFilename = null;
157-
if (project.getName().contains("oss-") || project.getName().equals("integ-test-zip")) {
158-
licenseFilename = "AGPL-3.0+SSPL-1.0+ELASTIC-LICENSE-2.0.txt";
159-
} else {
160-
licenseFilename = "ELASTIC-LICENSE-2.0.txt";
161-
}
162-
final List<String> licenseLines;
163-
try {
164-
licenseLines = Files.readAllLines(project.getRootDir().toPath().resolve("licenses/" + licenseFilename));
165-
final Path licensePath = checkExtraction.get()
166-
.getDestinationDir()
167-
.toPath()
168-
.resolve("elasticsearch-" + VersionProperties.getElasticsearch() + "/LICENSE.txt");
169-
assertLinesInFile(licensePath, licenseLines);
170-
} catch (IOException ioException) {
171-
ioException.printStackTrace();
172-
}
155+
String projectName = project.getName();
156+
Provider<Path> licensePathProvider = checkExtraction.map(
157+
copy -> copy.getDestinationDir().toPath().resolve("elasticsearch-" + VersionProperties.getElasticsearch() + "/LICENSE.txt")
158+
);
159+
File rootDir = project.getLayout().getSettingsDirectory().getAsFile();
160+
task.doLast(t -> {
161+
String licenseFilename = null;
162+
if (projectName.contains("oss-") || projectName.equals("integ-test-zip")) {
163+
licenseFilename = "AGPL-3.0+SSPL-1.0+ELASTIC-LICENSE-2.0.txt";
164+
} else {
165+
licenseFilename = "ELASTIC-LICENSE-2.0.txt";
166+
}
167+
final List<String> licenseLines;
168+
try {
169+
licenseLines = Files.readAllLines(rootDir.toPath().resolve("licenses/" + licenseFilename));
170+
assertLinesInFile(licensePathProvider.get(), licenseLines);
171+
} catch (IOException ioException) {
172+
ioException.printStackTrace();
173173
}
174174
});
175175
});

‎build-tools/src/main/java/org/elasticsearch/gradle/testclusters/RunTask.java‎

Lines changed: 4 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -42,7 +42,7 @@ public abstract class RunTask extends DefaultTestClustersTask {
4242

4343
private Boolean debug = false;
4444
private Boolean cliDebug = false;
45-
private Boolean entitlementsEnabled = false;
45+
4646
private Boolean apmServerEnabled = false;
4747

4848
private Boolean preserveData = false;
@@ -74,9 +74,7 @@ public void setCliDebug(boolean enabled) {
7474
option = "entitlements",
7575
description = "Use the Entitlements agent system in place of SecurityManager to enforce sandbox policies."
7676
)
77-
public void setEntitlementsEnabled(boolean enabled) {
78-
this.entitlementsEnabled = enabled;
79-
}
77+
public void setEntitlementsEnabled(boolean enabled) {}
8078

8179
@Input
8280
public Boolean getDebug() {
@@ -90,7 +88,7 @@ public Boolean getCliDebug() {
9088

9189
@Input
9290
public Boolean getEntitlementsEnabled() {
93-
return entitlementsEnabled;
91+
return true;
9492
}
9593

9694
@Input
@@ -240,9 +238,7 @@ else if (node.getSettingKeys().contains("telemetry.metrics.enabled") == false) {
240238
if (cliDebug) {
241239
enableCliDebug();
242240
}
243-
if (entitlementsEnabled) {
244-
enableEntitlements();
245-
}
241+
enableEntitlements();
246242
}
247243

248244
@TaskAction

‎distribution/src/config/log4j2.properties‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -129,3 +129,26 @@ logger.index_indexing_slowlog.name = index.indexing.slowlog.index
129129
logger.index_indexing_slowlog.level = trace
130130
logger.index_indexing_slowlog.appenderRef.index_indexing_slowlog_rolling.ref = index_indexing_slowlog_rolling
131131
logger.index_indexing_slowlog.additivity = false
132+
133+
134+
######## ES|QL query log JSON ####################
135+
appender.esql_querylog_rolling.type = RollingFile
136+
appender.esql_querylog_rolling.name = esql_querylog_rolling
137+
appender.esql_querylog_rolling.fileName = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs\
138+
.cluster_name}_esql_querylog.json
139+
appender.esql_querylog_rolling.layout.type = ECSJsonLayout
140+
appender.esql_querylog_rolling.layout.dataset = elasticsearch.esql_querylog
141+
142+
appender.esql_querylog_rolling.filePattern = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs\
143+
.cluster_name}_esql_querylog-%i.json.gz
144+
appender.esql_querylog_rolling.policies.type = Policies
145+
appender.esql_querylog_rolling.policies.size.type = SizeBasedTriggeringPolicy
146+
appender.esql_querylog_rolling.policies.size.size = 1GB
147+
appender.esql_querylog_rolling.strategy.type = DefaultRolloverStrategy
148+
appender.esql_querylog_rolling.strategy.max = 4
149+
#################################################
150+
151+
logger.esql_querylog_rolling.name = esql.querylog
152+
logger.esql_querylog_rolling.level = trace
153+
logger.esql_querylog_rolling.appenderRef.esql_querylog_rolling.ref = esql_querylog_rolling
154+
logger.esql_querylog_rolling.additivity = false

‎distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/SystemJvmOptions.java‎

Lines changed: 2 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,6 @@
1111

1212
import org.elasticsearch.common.settings.Settings;
1313
import org.elasticsearch.common.util.concurrent.EsExecutors;
14-
import org.elasticsearch.core.Booleans;
1514
import org.elasticsearch.jdk.RuntimeVersionFeature;
1615

1716
import java.io.IOException;
@@ -28,9 +27,8 @@ static List<String> systemJvmOptions(Settings nodeSettings, final Map<String, St
2827
String distroType = sysprops.get("es.distribution.type");
2928
String javaType = sysprops.get("es.java.type");
3029
boolean isHotspot = sysprops.getOrDefault("sun.management.compiler", "").contains("HotSpot");
31-
boolean entitlementsExplicitlyEnabled = Booleans.parseBoolean(sysprops.getOrDefault("es.entitlements.enabled", "true"));
32-
// java 24+ only supports entitlements, but it may be enabled on earlier versions explicitly
33-
boolean useEntitlements = RuntimeVersionFeature.isSecurityManagerAvailable() == false || entitlementsExplicitlyEnabled;
30+
31+
boolean useEntitlements = true;
3432
return Stream.of(
3533
Stream.of(
3634
/*

‎distribution/tools/server-cli/src/test/java/org/elasticsearch/server/cli/ServerCliTests.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -185,7 +185,7 @@ public void testElasticsearchSettingCanNotBeEmpty() throws Exception {
185185
}
186186

187187
public void testElasticsearchSettingCanNotBeDuplicated() throws Exception {
188-
assertUsage(containsString("setting [foo] already set, saw [bar] and [baz]"), "-E", "foo=bar", "-E", "foo=baz");
188+
assertUsage(containsString("setting [foo] set twice via command line -E"), "-E", "foo=bar", "-E", "foo=baz");
189189
}
190190

191191
public void testUnknownOption() throws Exception {

‎docs/changelog/120869.yaml‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
pr: 120869
2+
summary: Threadpool merge scheduler
3+
area: Engine
4+
type: feature
5+
issues: []

‎docs/changelog/121106.yaml‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
pr: 121106
2+
summary: Add `ModelRegistryMetadata` to Cluster State
3+
area: Machine Learning
4+
type: enhancement
5+
issues: []

‎docs/changelog/122062.yaml‎

Lines changed: 54 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,54 @@
1+
pr: 122062
2+
summary: Upgrade `discovery-ec2` to AWS SDK v2
3+
area: Discovery-Plugins
4+
type: breaking
5+
issues: []
6+
breaking:
7+
title: Upgrade `discovery-ec2` to AWS SDK v2
8+
area: Cluster and node setting
9+
details: >-
10+
11+
In earlier versions of {es} the `discovery-ec2` plugin was based on the AWS
12+
SDK v1. AWS will withdraw support for this SDK before the end of the life
13+
of {es} {minor-version} so we must migrate to the newer AWS SDK v2.
14+
15+
Unfortunately there are several differences between the two AWS SDK
16+
versions which may require you to adjust your system configuration when
17+
upgrading to {es} {minor-version} or later. These differences include, but
18+
may not be limited to, the following items.
19+
20+
* AWS SDK v2 does not support the EC2 IMDSv1 protocol.
21+
22+
* AWS SDK v2 does not support the `aws.secretKey` or
23+
`com.amazonaws.sdk.ec2MetadataServiceEndpointOverride` system properties.
24+
25+
* AWS SDK v2 does not permit specifying a choice between HTTP and HTTPS so
26+
the `discovery.ec2.protocol` setting is no longer effective.
27+
28+
* AWS SDK v2 does not accept an access key without a secret key or vice
29+
versa.
30+
31+
impact: >-
32+
33+
If you use the `discovery-ec2` plugin, test your upgrade thoroughly before
34+
upgrading any production workloads.
35+
36+
Adapt your configuration to the new SDK functionality. This includes, but
37+
may not be limited to, the following items.
38+
39+
* If you use IMDS to determine the availability zone of a node or to obtain
40+
credentials for accessing the EC2 API, ensure that it supports the IMDSv2
41+
protocol.
42+
43+
* If applicable, discontinue use of the `aws.secretKey` and
44+
`com.amazonaws.sdk.ec2MetadataServiceEndpointOverride` system properties.
45+
46+
* If applicable, specify that you wish to use the insecure HTTP protocol to
47+
access the EC2 API by setting `discovery.ec2.endpoint` to a URL which
48+
starts with `http://`.
49+
50+
* Either supply both an access key and a secret key using the keystore
51+
settings `discovery.ec2.access_key` and `discovery.ec2.secret_key`, or
52+
configure neither of these settings.
53+
54+
notable: true

‎docs/changelog/122638.yaml‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
pr: 122638
2+
summary: Expose `input_type` option at root level for `text_embedding` task type in
3+
Perform Inference API
4+
area: Machine Learning
5+
type: enhancement
6+
issues:
7+
- 117856

‎docs/changelog/124094.yaml‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
pr: 124094
2+
summary: ES|QL slow log
3+
area: ES|QL
4+
type: enhancement
5+
issues: []

0 commit comments

Comments
 (0)