Skip to content

Commit a3f87df

Browse files
albertzaharovitsalbertzaharovits
committed
Merge branch 'main' into threadpool-merge-scheduler
2 parents bf557d2 + be049bb commit a3f87df

File tree

325 files changed

+5790
-2207
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

325 files changed

+5790
-2207
lines changed

‎benchmarks/build.gradle‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
import org.elasticsearch.gradle.internal.test.TestUtil
2+
import org.elasticsearch.gradle.OS
23

34
/*
45
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
@@ -77,7 +78,7 @@ tasks.register("copyPainless", Copy) {
7778
}
7879

7980
tasks.named("run").configure {
80-
executable = "${buildParams.runtimeJavaHome.get()}/bin/java"
81+
executable = "${buildParams.runtimeJavaHome.get()}/bin/java" + (OS.current() == OS.WINDOWS ? '.exe' : '')
8182
args << "-Dplugins.dir=${buildDir}/plugins" << "-Dtests.index=${buildDir}/index"
8283
dependsOn "copyExpression", "copyPainless", configurations.nativeLib
8384
systemProperty 'es.nativelibs.path', TestUtil.getTestLibraryPath(file("../libs/native/libraries/build/platform/").toString())

‎build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/DockerBase.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,7 @@ public enum DockerBase {
2222
// Chainguard based wolfi image with latest jdk
2323
// This is usually updated via renovatebot
2424
// spotless:off
25-
WOLFI("docker.elastic.co/wolfi/chainguard-base:latest@sha256:eef54b3a414aa53b98f0f8df2633aed83c3ba6230722769282925442968f0364",
25+
WOLFI("docker.elastic.co/wolfi/chainguard-base:latest@sha256:dd66beec64a7f9b19c6c35a1195153b2b630a55e16ec71949ed5187c5947eea1",
2626
"-wolfi",
2727
"apk"
2828
),

‎build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/testfixtures/TestFixturesPlugin.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -122,7 +122,7 @@ public void apply(Project project) {
122122
composeExtension.getRemoveContainers().set(true);
123123
composeExtension.getCaptureContainersOutput()
124124
.set(EnumSet.of(LogLevel.INFO, LogLevel.DEBUG).contains(project.getGradle().getStartParameter().getLogLevel()));
125-
composeExtension.getUseDockerComposeV2().set(false);
125+
composeExtension.getUseDockerComposeV2().set(true);
126126
composeExtension.getExecutable().set(this.providerFactory.provider(() -> {
127127
String composePath = dockerSupport.get().getDockerAvailability().dockerComposePath();
128128
LOGGER.debug("Docker Compose path: {}", composePath);

‎build-tools/src/main/java/org/elasticsearch/gradle/testclusters/TestClustersAware.java‎

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,7 @@ public interface TestClustersAware extends Task {
2626
Collection<ElasticsearchCluster> getClusters();
2727

2828
@ServiceReference(REGISTRY_SERVICE_NAME)
29-
Property<TestClustersRegistry> getRegistery();
29+
Property<TestClustersRegistry> getRegistry();
3030

3131
@ServiceReference(TEST_CLUSTER_TASKS_SERVICE)
3232
Property<TestClustersPlugin.TaskEventsService> getTasksService();
@@ -47,6 +47,14 @@ default void useCluster(ElasticsearchCluster cluster) {
4747
getClusters().add(cluster);
4848
}
4949

50+
default Provider<TestClusterInfo> getClusterInfo(String clusterName) {
51+
return getProject().getProviders().of(TestClusterValueSource.class, source -> {
52+
source.getParameters().getService().set(getRegistry());
53+
source.getParameters().getClusterName().set(clusterName);
54+
source.getParameters().getPath().set(getProject().getIsolated().getPath());
55+
});
56+
}
57+
5058
default void useCluster(Provider<ElasticsearchCluster> cluster) {
5159
useCluster(cluster.get());
5260
}

‎build-tools/src/main/java/org/elasticsearch/gradle/testclusters/TestClustersPlugin.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -249,7 +249,7 @@ private void configureStartClustersHook(Gradle gradle) {
249249
.forEach(awareTask -> {
250250
awareTask.doFirst(task -> {
251251
awareTask.beforeStart();
252-
awareTask.getClusters().forEach(awareTask.getRegistery().get()::maybeStartCluster);
252+
awareTask.getClusters().forEach(awareTask.getRegistry().get()::maybeStartCluster);
253253
});
254254
});
255255
});

‎build-tools/src/main/java/org/elasticsearch/gradle/testclusters/TestClustersRegistry.java‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -109,6 +109,23 @@ public void restart(String path, String clusterName) {
109109
cluster.restart();
110110
}
111111

112+
public void nextNodeToNextVersion(Provider<ElasticsearchCluster> cluster) {
113+
nextNodeToNextVersion(cluster.get());
114+
}
115+
116+
public void nextNodeToNextVersion(ElasticsearchCluster cluster) {
117+
nextNodeToNextVersion(cluster.getPath(), cluster.getName());
118+
}
119+
120+
public void nextNodeToNextVersion(String path, String clusterName) {
121+
ElasticsearchCluster cluster = runningClusters.stream()
122+
.filter(c -> c.getPath().equals(path))
123+
.filter(c -> c.getName().equals(clusterName))
124+
.findFirst()
125+
.orElseThrow();
126+
cluster.nextNodeToNextVersion();
127+
}
128+
112129
public void storeProcess(String id, Process esProcess) {
113130
nodeProcesses.put(id, esProcess);
114131
}

‎distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/APMJvmOptions.java‎

Lines changed: 45 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -187,12 +187,20 @@ static String agentCommandLineOption(Path agentJar, Path tmpPropertiesFile) {
187187
static void extractSecureSettings(SecureSettings secrets, Map<String, String> propertiesMap) {
188188
final Set<String> settingNames = secrets.getSettingNames();
189189
for (String key : List.of("api_key", "secret_token")) {
190-
String prefix = "telemetry.";
191-
if (settingNames.contains(prefix + key)) {
192-
try (SecureString token = secrets.getString(prefix + key)) {
193-
propertiesMap.put(key, token.toString());
190+
for (String prefix : List.of("telemetry.", "tracing.apm.")) {
191+
if (settingNames.contains(prefix + key)) {
192+
if (propertiesMap.containsKey(key)) {
193+
throw new IllegalStateException(
194+
Strings.format("Duplicate telemetry setting: [telemetry.%s] and [tracing.apm.%s]", key, key)
195+
);
196+
}
197+
198+
try (SecureString token = secrets.getString(prefix + key)) {
199+
propertiesMap.put(key, token.toString());
200+
}
194201
}
195202
}
203+
196204
}
197205
}
198206

@@ -219,12 +227,44 @@ private static Map<String, String> extractDynamicSettings(Map<String, String> pr
219227
static Map<String, String> extractApmSettings(Settings settings) throws UserException {
220228
final Map<String, String> propertiesMap = new HashMap<>();
221229

230+
// tracing.apm.agent. is deprecated by telemetry.agent.
222231
final String telemetryAgentPrefix = "telemetry.agent.";
232+
final String deprecatedTelemetryAgentPrefix = "tracing.apm.agent.";
223233

224234
final Settings telemetryAgentSettings = settings.getByPrefix(telemetryAgentPrefix);
225235
telemetryAgentSettings.keySet().forEach(key -> propertiesMap.put(key, String.valueOf(telemetryAgentSettings.get(key))));
226236

237+
final Settings apmAgentSettings = settings.getByPrefix(deprecatedTelemetryAgentPrefix);
238+
for (String key : apmAgentSettings.keySet()) {
239+
if (propertiesMap.containsKey(key)) {
240+
throw new IllegalStateException(
241+
Strings.format(
242+
"Duplicate telemetry setting: [%s%s] and [%s%s]",
243+
telemetryAgentPrefix,
244+
key,
245+
deprecatedTelemetryAgentPrefix,
246+
key
247+
)
248+
);
249+
}
250+
propertiesMap.put(key, String.valueOf(apmAgentSettings.get(key)));
251+
}
252+
227253
StringJoiner globalLabels = extractGlobalLabels(telemetryAgentPrefix, propertiesMap, settings);
254+
if (globalLabels.length() == 0) {
255+
globalLabels = extractGlobalLabels(deprecatedTelemetryAgentPrefix, propertiesMap, settings);
256+
} else {
257+
StringJoiner tracingGlobalLabels = extractGlobalLabels(deprecatedTelemetryAgentPrefix, propertiesMap, settings);
258+
if (tracingGlobalLabels.length() != 0) {
259+
throw new IllegalArgumentException(
260+
"Cannot have global labels with tracing.agent prefix ["
261+
+ globalLabels
262+
+ "] and telemetry.apm.agent prefix ["
263+
+ tracingGlobalLabels
264+
+ "]"
265+
);
266+
}
267+
}
228268
if (globalLabels.length() > 0) {
229269
propertiesMap.put("global_labels", globalLabels.toString());
230270
}
@@ -234,7 +274,7 @@ static Map<String, String> extractApmSettings(Settings settings) throws UserExce
234274
if (propertiesMap.containsKey(key)) {
235275
throw new UserException(
236276
ExitCodes.CONFIG,
237-
"Do not set a value for [telemetry.agent." + key + "], as this is configured automatically by Elasticsearch"
277+
"Do not set a value for [tracing.apm.agent." + key + "], as this is configured automatically by Elasticsearch"
238278
);
239279
}
240280
}

‎distribution/tools/server-cli/src/test/java/org/elasticsearch/server/cli/APMJvmOptionsTests.java‎

Lines changed: 96 additions & 47 deletions
Original file line numberDiff line numberDiff line change
@@ -25,15 +25,18 @@
2525
import java.util.HashMap;
2626
import java.util.List;
2727
import java.util.Map;
28+
import java.util.function.Function;
2829

2930
import static org.elasticsearch.test.MapMatcher.matchesMap;
3031
import static org.hamcrest.Matchers.allOf;
3132
import static org.hamcrest.Matchers.containsInAnyOrder;
33+
import static org.hamcrest.Matchers.containsString;
3234
import static org.hamcrest.Matchers.endsWith;
3335
import static org.hamcrest.Matchers.equalTo;
3436
import static org.hamcrest.Matchers.hasEntry;
3537
import static org.hamcrest.Matchers.hasKey;
3638
import static org.hamcrest.Matchers.hasSize;
39+
import static org.hamcrest.Matchers.is;
3740
import static org.hamcrest.Matchers.not;
3841
import static org.mockito.Mockito.doReturn;
3942
import static org.mockito.Mockito.mock;
@@ -79,63 +82,109 @@ public void testFileDeleteWorks() throws IOException {
7982
}
8083

8184
public void testExtractSecureSettings() {
82-
MockSecureSettings secureSettings = new MockSecureSettings();
83-
secureSettings.setString("telemetry.secret_token", "token");
84-
secureSettings.setString("telemetry.api_key", "key");
85+
MockSecureSettings duplicateSecureSettings = new MockSecureSettings();
8586

86-
Map<String, String> propertiesMap = new HashMap<>();
87-
APMJvmOptions.extractSecureSettings(secureSettings, propertiesMap);
87+
for (String prefix : List.of("telemetry.", "tracing.apm.")) {
88+
MockSecureSettings secureSettings = new MockSecureSettings();
89+
secureSettings.setString(prefix + "secret_token", "token");
90+
secureSettings.setString(prefix + "api_key", "key");
8891

89-
assertThat(propertiesMap, matchesMap(Map.of("secret_token", "token", "api_key", "key")));
90-
}
92+
duplicateSecureSettings.setString(prefix + "api_key", "secret");
9193

92-
public void testExtractSettings() throws UserException {
93-
Settings defaults = Settings.builder()
94-
.put("telemetry.agent.server_url", "https://myurl:443")
95-
.put("telemetry.agent.service_node_name", "instance-0000000001")
96-
.build();
94+
Map<String, String> propertiesMap = new HashMap<>();
95+
APMJvmOptions.extractSecureSettings(secureSettings, propertiesMap);
96+
97+
assertThat(propertiesMap, matchesMap(Map.of("secret_token", "token", "api_key", "key")));
98+
}
9799

98-
var name = "APM Tracing";
99-
var deploy = "123";
100-
var org = "456";
101-
var extracted = APMJvmOptions.extractApmSettings(
102-
Settings.builder()
103-
.put(defaults)
104-
.put("telemetry.agent.global_labels.deployment_name", name)
105-
.put("telemetry.agent.global_labels.deployment_id", deploy)
106-
.put("telemetry.agent.global_labels.organization_id", org)
107-
.build()
100+
Exception exception = expectThrows(
101+
IllegalStateException.class,
102+
() -> APMJvmOptions.extractSecureSettings(duplicateSecureSettings, new HashMap<>())
108103
);
104+
assertThat(exception.getMessage(), containsString("Duplicate telemetry setting"));
105+
assertThat(exception.getMessage(), containsString("telemetry.api_key"));
106+
assertThat(exception.getMessage(), containsString("tracing.apm.api_key"));
109107

110-
assertThat(
111-
extracted,
112-
allOf(
113-
hasEntry("server_url", "https://myurl:443"),
114-
hasEntry("service_node_name", "instance-0000000001"),
115-
hasEntry(equalTo("global_labels"), not(endsWith(","))), // test that we have collapsed all global labels into one
116-
not(hasKey("global_labels.organization_id")) // tests that we strip out the top level label keys
108+
}
109+
110+
public void testExtractSettings() throws UserException {
111+
Function<String, Settings.Builder> buildSettings = (prefix) -> Settings.builder()
112+
.put(prefix + "server_url", "https://myurl:443")
113+
.put(prefix + "service_node_name", "instance-0000000001");
114+
115+
for (String prefix : List.of("tracing.apm.agent.", "telemetry.agent.")) {
116+
var name = "APM Tracing";
117+
var deploy = "123";
118+
var org = "456";
119+
var extracted = APMJvmOptions.extractApmSettings(
120+
buildSettings.apply(prefix)
121+
.put(prefix + "global_labels.deployment_name", name)
122+
.put(prefix + "global_labels.deployment_id", deploy)
123+
.put(prefix + "global_labels.organization_id", org)
124+
.build()
125+
);
126+
127+
assertThat(
128+
extracted,
129+
allOf(
130+
hasEntry("server_url", "https://myurl:443"),
131+
hasEntry("service_node_name", "instance-0000000001"),
132+
hasEntry(equalTo("global_labels"), not(endsWith(","))), // test that we have collapsed all global labels into one
133+
not(hasKey("global_labels.organization_id")) // tests that we strip out the top level label keys
134+
)
135+
);
136+
137+
List<String> labels = Arrays.stream(extracted.get("global_labels").split(",")).toList();
138+
assertThat(labels, hasSize(3));
139+
assertThat(labels, containsInAnyOrder("deployment_name=APM Tracing", "organization_id=" + org, "deployment_id=" + deploy));
140+
141+
// test replacing with underscores and skipping empty
142+
name = "APM=Tracing";
143+
deploy = "";
144+
org = ",456";
145+
extracted = APMJvmOptions.extractApmSettings(
146+
buildSettings.apply(prefix)
147+
.put(prefix + "global_labels.deployment_name", name)
148+
.put(prefix + "global_labels.deployment_id", deploy)
149+
.put(prefix + "global_labels.organization_id", org)
150+
.build()
151+
);
152+
labels = Arrays.stream(extracted.get("global_labels").split(",")).toList();
153+
assertThat(labels, hasSize(2));
154+
assertThat(labels, containsInAnyOrder("deployment_name=APM_Tracing", "organization_id=_456"));
155+
}
156+
157+
IllegalStateException err = expectThrows(
158+
IllegalStateException.class,
159+
() -> APMJvmOptions.extractApmSettings(
160+
Settings.builder()
161+
.put("tracing.apm.agent.server_url", "https://myurl:443")
162+
.put("telemetry.agent.server_url", "https://myurl-2:443")
163+
.build()
117164
)
118165
);
166+
assertThat(err.getMessage(), is("Duplicate telemetry setting: [telemetry.agent.server_url] and [tracing.apm.agent.server_url]"));
167+
}
168+
169+
public void testNoMixedLabels() {
170+
String telemetryAgent = "telemetry.agent.";
171+
String tracingAgent = "tracing.apm.agent.";
172+
Settings settings = Settings.builder()
173+
.put("tracing.apm.enabled", true)
174+
.put(telemetryAgent + "server_url", "https://myurl:443")
175+
.put(telemetryAgent + "service_node_name", "instance-0000000001")
176+
.put(tracingAgent + "global_labels.deployment_id", "123")
177+
.put(telemetryAgent + "global_labels.organization_id", "456")
178+
.build();
119179

120-
List<String> labels = Arrays.stream(extracted.get("global_labels").split(",")).toList();
121-
assertThat(labels, hasSize(3));
122-
assertThat(labels, containsInAnyOrder("deployment_name=APM Tracing", "organization_id=" + org, "deployment_id=" + deploy));
123-
124-
// test replacing with underscores and skipping empty
125-
name = "APM=Tracing";
126-
deploy = "";
127-
org = ",456";
128-
extracted = APMJvmOptions.extractApmSettings(
129-
Settings.builder()
130-
.put(defaults)
131-
.put("telemetry.agent.global_labels.deployment_name", name)
132-
.put("telemetry.agent.global_labels.deployment_id", deploy)
133-
.put("telemetry.agent.global_labels.organization_id", org)
134-
.build()
180+
IllegalArgumentException err = assertThrows(IllegalArgumentException.class, () -> APMJvmOptions.extractApmSettings(settings));
181+
assertThat(
182+
err.getMessage(),
183+
is(
184+
"Cannot have global labels with tracing.agent prefix [organization_id=456] and"
185+
+ " telemetry.apm.agent prefix [deployment_id=123]"
186+
)
135187
);
136-
labels = Arrays.stream(extracted.get("global_labels").split(",")).toList();
137-
assertThat(labels, hasSize(2));
138-
assertThat(labels, containsInAnyOrder("deployment_name=APM_Tracing", "organization_id=_456"));
139188
}
140189

141190
private Path makeFakeAgentJar() throws IOException {

‎docs/changelog/118188.yaml‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
pr: 118188
2+
summary: Check for early termination in Driver
3+
area: ES|QL
4+
type: enhancement
5+
issues: []

‎docs/changelog/119575.yaml‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
pr: 119575
2+
summary: Fix realtime get of nested fields with synthetic source
3+
area: Mapping
4+
type: bug
5+
issues:
6+
- 119553

0 commit comments

Comments
 (0)