Skip to content

Commit 5d3aeb4

Browse files
ChrisHegartyChrisHegarty
committed
Merge branch 'main' into lucene_snapshot
2 parents f5a7192 + 560ecf0 commit 5d3aeb4

File tree

613 files changed

+15796
-7107
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

613 files changed

+15796
-7107
lines changed

‎.buildkite/pipelines/pull-request/part-4-entitlements.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,5 +7,5 @@ steps:
77
agents:
88
provider: gcp
99
image: family/elasticsearch-ubuntu-2004
10-
machineType: custom-32-98304
10+
machineType: n1-standard-32
1111
buildDirectory: /dev/shm/bk

‎.buildkite/pipelines/pull-request/part-4-fips.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,5 +7,5 @@ steps:
77
agents:
88
provider: gcp
99
image: family/elasticsearch-ubuntu-2004
10-
machineType: custom-32-98304
10+
machineType: n1-standard-32
1111
buildDirectory: /dev/shm/bk

‎.buildkite/pipelines/pull-request/part-4.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,5 +7,5 @@ steps:
77
agents:
88
provider: gcp
99
image: family/elasticsearch-ubuntu-2004
10-
machineType: custom-32-98304
10+
machineType: n1-standard-32
1111
buildDirectory: /dev/shm/bk

‎benchmarks/src/main/java/org/elasticsearch/benchmark/search/aggregations/TermsReduceBenchmark.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -111,7 +111,7 @@ public void setup() {
111111
dict[i] = new BytesRef(Long.toString(rand.nextLong()));
112112
}
113113
for (int i = 0; i < numShards; i++) {
114-
aggsList.add(InternalAggregations.from(Collections.singletonList(newTerms(rand, dict, true))));
114+
aggsList.add(InternalAggregations.from(newTerms(rand, dict, true)));
115115
}
116116
}
117117

@@ -124,7 +124,7 @@ private StringTerms newTerms(Random rand, BytesRef[] dict, boolean withNested) {
124124
for (BytesRef term : randomTerms) {
125125
InternalAggregations subAggs;
126126
if (withNested) {
127-
subAggs = InternalAggregations.from(Collections.singletonList(newTerms(rand, dict, false)));
127+
subAggs = InternalAggregations.from(newTerms(rand, dict, false));
128128
} else {
129129
subAggs = InternalAggregations.EMPTY;
130130
}

‎benchmarks/src/main/java/org/elasticsearch/benchmark/search/aggregations/bucket/terms/StringTermsSerializationBenchmark.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -50,13 +50,13 @@ public class StringTermsSerializationBenchmark {
5050

5151
@Setup
5252
public void initResults() {
53-
results = DelayableWriteable.referencing(InternalAggregations.from(List.of(newTerms(true))));
53+
results = DelayableWriteable.referencing(InternalAggregations.from(newTerms(true)));
5454
}
5555

5656
private StringTerms newTerms(boolean withNested) {
5757
List<StringTerms.Bucket> resultBuckets = new ArrayList<>(buckets);
5858
for (int i = 0; i < buckets; i++) {
59-
InternalAggregations inner = withNested ? InternalAggregations.from(List.of(newTerms(false))) : InternalAggregations.EMPTY;
59+
InternalAggregations inner = withNested ? InternalAggregations.from(newTerms(false)) : InternalAggregations.EMPTY;
6060
resultBuckets.add(new StringTerms.Bucket(new BytesRef("test" + i), i, inner, false, 0, DocValueFormat.RAW));
6161
}
6262
return new StringTerms(

‎build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/MrjarPlugin.java‎

Lines changed: 23 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -72,18 +72,19 @@ public void apply(Project project) {
7272
var javaExtension = project.getExtensions().getByType(JavaPluginExtension.class);
7373
var isIdeaSync = System.getProperty("idea.sync.active", "false").equals("true");
7474
var ideaSourceSetsEnabled = project.hasProperty(MRJAR_IDEA_ENABLED) && project.property(MRJAR_IDEA_ENABLED).equals("true");
75+
int minJavaVersion = Integer.parseInt(buildParams.getMinimumCompilerVersion().getMajorVersion());
7576

7677
// Ignore version-specific source sets if we are importing into IntelliJ and have not explicitly enabled this.
7778
// Avoids an IntelliJ bug:
7879
// https://youtrack.jetbrains.com/issue/IDEA-285640/Compiler-Options-Settings-language-level-is-set-incorrectly-with-JDK-19ea
7980
if (isIdeaSync == false || ideaSourceSetsEnabled) {
80-
List<Integer> mainVersions = findSourceVersions(project);
81+
List<Integer> mainVersions = findSourceVersions(project, minJavaVersion);
8182
List<String> mainSourceSets = new ArrayList<>();
8283
mainSourceSets.add(SourceSet.MAIN_SOURCE_SET_NAME);
83-
configurePreviewFeatures(project, javaExtension.getSourceSets().getByName(SourceSet.MAIN_SOURCE_SET_NAME), 21);
84+
configurePreviewFeatures(project, javaExtension.getSourceSets().getByName(SourceSet.MAIN_SOURCE_SET_NAME), minJavaVersion);
8485
List<String> testSourceSets = new ArrayList<>(mainSourceSets);
8586
testSourceSets.add(SourceSet.TEST_SOURCE_SET_NAME);
86-
configurePreviewFeatures(project, javaExtension.getSourceSets().getByName(SourceSet.TEST_SOURCE_SET_NAME), 21);
87+
configurePreviewFeatures(project, javaExtension.getSourceSets().getByName(SourceSet.TEST_SOURCE_SET_NAME), minJavaVersion);
8788
for (int javaVersion : mainVersions) {
8889
String mainSourceSetName = SourceSet.MAIN_SOURCE_SET_NAME + javaVersion;
8990
SourceSet mainSourceSet = addSourceSet(project, javaExtension, mainSourceSetName, mainSourceSets, javaVersion, true);
@@ -103,6 +104,7 @@ public void apply(Project project) {
103104
}
104105

105106
private void configureMrjar(Project project) {
107+
106108
var jarTask = project.getTasks().withType(Jar.class).named(JavaPlugin.JAR_TASK_NAME);
107109
jarTask.configure(task -> { task.manifest(manifest -> { manifest.attributes(Map.of("Multi-Release", "true")); }); });
108110

@@ -222,7 +224,7 @@ private void createTestTask(
222224
project.getTasks().named("check").configure(checkTask -> checkTask.dependsOn(testTaskProvider));
223225
}
224226

225-
private static List<Integer> findSourceVersions(Project project) {
227+
private static List<Integer> findSourceVersions(Project project, int minJavaVersion) {
226228
var srcDir = project.getProjectDir().toPath().resolve("src");
227229
List<Integer> versions = new ArrayList<>();
228230
try (var subdirStream = Files.list(srcDir)) {
@@ -231,7 +233,23 @@ private static List<Integer> findSourceVersions(Project project) {
231233
String sourcesetName = sourceSetPath.getFileName().toString();
232234
Matcher sourcesetMatcher = MRJAR_SOURCESET_PATTERN.matcher(sourcesetName);
233235
if (sourcesetMatcher.matches()) {
234-
versions.add(Integer.parseInt(sourcesetMatcher.group(1)));
236+
int version = Integer.parseInt(sourcesetMatcher.group(1));
237+
if (version < minJavaVersion) {
238+
// NOTE: We allow mainNN for the min java version so that incubating modules can be used without warnings.
239+
// It is a workaround for https://bugs.openjdk.org/browse/JDK-8187591. Once min java is 22, we
240+
// can use the SuppressWarnings("preview") in the code using incubating modules and this check
241+
// can change to <=
242+
throw new IllegalArgumentException(
243+
"Found src dir '"
244+
+ sourcesetName
245+
+ "' for Java "
246+
+ version
247+
+ " but multi-release jar sourceset should have version "
248+
+ minJavaVersion
249+
+ " or greater"
250+
);
251+
}
252+
versions.add(version);
235253
}
236254
}
237255
} catch (IOException e) {

‎distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/APMJvmOptions.java‎

Lines changed: 5 additions & 45 deletions
Original file line numberDiff line numberDiff line change
@@ -187,20 +187,12 @@ static String agentCommandLineOption(Path agentJar, Path tmpPropertiesFile) {
187187
static void extractSecureSettings(SecureSettings secrets, Map<String, String> propertiesMap) {
188188
final Set<String> settingNames = secrets.getSettingNames();
189189
for (String key : List.of("api_key", "secret_token")) {
190-
for (String prefix : List.of("telemetry.", "tracing.apm.")) {
191-
if (settingNames.contains(prefix + key)) {
192-
if (propertiesMap.containsKey(key)) {
193-
throw new IllegalStateException(
194-
Strings.format("Duplicate telemetry setting: [telemetry.%s] and [tracing.apm.%s]", key, key)
195-
);
196-
}
197-
198-
try (SecureString token = secrets.getString(prefix + key)) {
199-
propertiesMap.put(key, token.toString());
200-
}
190+
String prefix = "telemetry.";
191+
if (settingNames.contains(prefix + key)) {
192+
try (SecureString token = secrets.getString(prefix + key)) {
193+
propertiesMap.put(key, token.toString());
201194
}
202195
}
203-
204196
}
205197
}
206198

@@ -227,44 +219,12 @@ private static Map<String, String> extractDynamicSettings(Map<String, String> pr
227219
static Map<String, String> extractApmSettings(Settings settings) throws UserException {
228220
final Map<String, String> propertiesMap = new HashMap<>();
229221

230-
// tracing.apm.agent. is deprecated by telemetry.agent.
231222
final String telemetryAgentPrefix = "telemetry.agent.";
232-
final String deprecatedTelemetryAgentPrefix = "tracing.apm.agent.";
233223

234224
final Settings telemetryAgentSettings = settings.getByPrefix(telemetryAgentPrefix);
235225
telemetryAgentSettings.keySet().forEach(key -> propertiesMap.put(key, String.valueOf(telemetryAgentSettings.get(key))));
236226

237-
final Settings apmAgentSettings = settings.getByPrefix(deprecatedTelemetryAgentPrefix);
238-
for (String key : apmAgentSettings.keySet()) {
239-
if (propertiesMap.containsKey(key)) {
240-
throw new IllegalStateException(
241-
Strings.format(
242-
"Duplicate telemetry setting: [%s%s] and [%s%s]",
243-
telemetryAgentPrefix,
244-
key,
245-
deprecatedTelemetryAgentPrefix,
246-
key
247-
)
248-
);
249-
}
250-
propertiesMap.put(key, String.valueOf(apmAgentSettings.get(key)));
251-
}
252-
253227
StringJoiner globalLabels = extractGlobalLabels(telemetryAgentPrefix, propertiesMap, settings);
254-
if (globalLabels.length() == 0) {
255-
globalLabels = extractGlobalLabels(deprecatedTelemetryAgentPrefix, propertiesMap, settings);
256-
} else {
257-
StringJoiner tracingGlobalLabels = extractGlobalLabels(deprecatedTelemetryAgentPrefix, propertiesMap, settings);
258-
if (tracingGlobalLabels.length() != 0) {
259-
throw new IllegalArgumentException(
260-
"Cannot have global labels with tracing.agent prefix ["
261-
+ globalLabels
262-
+ "] and telemetry.apm.agent prefix ["
263-
+ tracingGlobalLabels
264-
+ "]"
265-
);
266-
}
267-
}
268228
if (globalLabels.length() > 0) {
269229
propertiesMap.put("global_labels", globalLabels.toString());
270230
}
@@ -274,7 +234,7 @@ static Map<String, String> extractApmSettings(Settings settings) throws UserExce
274234
if (propertiesMap.containsKey(key)) {
275235
throw new UserException(
276236
ExitCodes.CONFIG,
277-
"Do not set a value for [tracing.apm.agent." + key + "], as this is configured automatically by Elasticsearch"
237+
"Do not set a value for [telemetry.agent." + key + "], as this is configured automatically by Elasticsearch"
278238
);
279239
}
280240
}

‎distribution/tools/server-cli/src/test/java/org/elasticsearch/server/cli/APMJvmOptionsTests.java‎

Lines changed: 47 additions & 96 deletions
Original file line numberDiff line numberDiff line change
@@ -25,18 +25,15 @@
2525
import java.util.HashMap;
2626
import java.util.List;
2727
import java.util.Map;
28-
import java.util.function.Function;
2928

3029
import static org.elasticsearch.test.MapMatcher.matchesMap;
3130
import static org.hamcrest.Matchers.allOf;
3231
import static org.hamcrest.Matchers.containsInAnyOrder;
33-
import static org.hamcrest.Matchers.containsString;
3432
import static org.hamcrest.Matchers.endsWith;
3533
import static org.hamcrest.Matchers.equalTo;
3634
import static org.hamcrest.Matchers.hasEntry;
3735
import static org.hamcrest.Matchers.hasKey;
3836
import static org.hamcrest.Matchers.hasSize;
39-
import static org.hamcrest.Matchers.is;
4037
import static org.hamcrest.Matchers.not;
4138
import static org.mockito.Mockito.doReturn;
4239
import static org.mockito.Mockito.mock;
@@ -82,109 +79,63 @@ public void testFileDeleteWorks() throws IOException {
8279
}
8380

8481
public void testExtractSecureSettings() {
85-
MockSecureSettings duplicateSecureSettings = new MockSecureSettings();
82+
MockSecureSettings secureSettings = new MockSecureSettings();
83+
secureSettings.setString("telemetry.secret_token", "token");
84+
secureSettings.setString("telemetry.api_key", "key");
8685

87-
for (String prefix : List.of("telemetry.", "tracing.apm.")) {
88-
MockSecureSettings secureSettings = new MockSecureSettings();
89-
secureSettings.setString(prefix + "secret_token", "token");
90-
secureSettings.setString(prefix + "api_key", "key");
91-
92-
duplicateSecureSettings.setString(prefix + "api_key", "secret");
93-
94-
Map<String, String> propertiesMap = new HashMap<>();
95-
APMJvmOptions.extractSecureSettings(secureSettings, propertiesMap);
96-
97-
assertThat(propertiesMap, matchesMap(Map.of("secret_token", "token", "api_key", "key")));
98-
}
99-
100-
Exception exception = expectThrows(
101-
IllegalStateException.class,
102-
() -> APMJvmOptions.extractSecureSettings(duplicateSecureSettings, new HashMap<>())
103-
);
104-
assertThat(exception.getMessage(), containsString("Duplicate telemetry setting"));
105-
assertThat(exception.getMessage(), containsString("telemetry.api_key"));
106-
assertThat(exception.getMessage(), containsString("tracing.apm.api_key"));
86+
Map<String, String> propertiesMap = new HashMap<>();
87+
APMJvmOptions.extractSecureSettings(secureSettings, propertiesMap);
10788

89+
assertThat(propertiesMap, matchesMap(Map.of("secret_token", "token", "api_key", "key")));
10890
}
10991

11092
public void testExtractSettings() throws UserException {
111-
Function<String, Settings.Builder> buildSettings = (prefix) -> Settings.builder()
112-
.put(prefix + "server_url", "https://myurl:443")
113-
.put(prefix + "service_node_name", "instance-0000000001");
114-
115-
for (String prefix : List.of("tracing.apm.agent.", "telemetry.agent.")) {
116-
var name = "APM Tracing";
117-
var deploy = "123";
118-
var org = "456";
119-
var extracted = APMJvmOptions.extractApmSettings(
120-
buildSettings.apply(prefix)
121-
.put(prefix + "global_labels.deployment_name", name)
122-
.put(prefix + "global_labels.deployment_id", deploy)
123-
.put(prefix + "global_labels.organization_id", org)
124-
.build()
125-
);
126-
127-
assertThat(
128-
extracted,
129-
allOf(
130-
hasEntry("server_url", "https://myurl:443"),
131-
hasEntry("service_node_name", "instance-0000000001"),
132-
hasEntry(equalTo("global_labels"), not(endsWith(","))), // test that we have collapsed all global labels into one
133-
not(hasKey("global_labels.organization_id")) // tests that we strip out the top level label keys
134-
)
135-
);
136-
137-
List<String> labels = Arrays.stream(extracted.get("global_labels").split(",")).toList();
138-
assertThat(labels, hasSize(3));
139-
assertThat(labels, containsInAnyOrder("deployment_name=APM Tracing", "organization_id=" + org, "deployment_id=" + deploy));
140-
141-
// test replacing with underscores and skipping empty
142-
name = "APM=Tracing";
143-
deploy = "";
144-
org = ",456";
145-
extracted = APMJvmOptions.extractApmSettings(
146-
buildSettings.apply(prefix)
147-
.put(prefix + "global_labels.deployment_name", name)
148-
.put(prefix + "global_labels.deployment_id", deploy)
149-
.put(prefix + "global_labels.organization_id", org)
150-
.build()
151-
);
152-
labels = Arrays.stream(extracted.get("global_labels").split(",")).toList();
153-
assertThat(labels, hasSize(2));
154-
assertThat(labels, containsInAnyOrder("deployment_name=APM_Tracing", "organization_id=_456"));
155-
}
156-
157-
IllegalStateException err = expectThrows(
158-
IllegalStateException.class,
159-
() -> APMJvmOptions.extractApmSettings(
160-
Settings.builder()
161-
.put("tracing.apm.agent.server_url", "https://myurl:443")
162-
.put("telemetry.agent.server_url", "https://myurl-2:443")
163-
.build()
164-
)
165-
);
166-
assertThat(err.getMessage(), is("Duplicate telemetry setting: [telemetry.agent.server_url] and [tracing.apm.agent.server_url]"));
167-
}
168-
169-
public void testNoMixedLabels() {
170-
String telemetryAgent = "telemetry.agent.";
171-
String tracingAgent = "tracing.apm.agent.";
172-
Settings settings = Settings.builder()
173-
.put("tracing.apm.enabled", true)
174-
.put(telemetryAgent + "server_url", "https://myurl:443")
175-
.put(telemetryAgent + "service_node_name", "instance-0000000001")
176-
.put(tracingAgent + "global_labels.deployment_id", "123")
177-
.put(telemetryAgent + "global_labels.organization_id", "456")
93+
Settings defaults = Settings.builder()
94+
.put("telemetry.agent.server_url", "https://myurl:443")
95+
.put("telemetry.agent.service_node_name", "instance-0000000001")
17896
.build();
17997

180-
IllegalArgumentException err = assertThrows(IllegalArgumentException.class, () -> APMJvmOptions.extractApmSettings(settings));
98+
var name = "APM Tracing";
99+
var deploy = "123";
100+
var org = "456";
101+
var extracted = APMJvmOptions.extractApmSettings(
102+
Settings.builder()
103+
.put(defaults)
104+
.put("telemetry.agent.global_labels.deployment_name", name)
105+
.put("telemetry.agent.global_labels.deployment_id", deploy)
106+
.put("telemetry.agent.global_labels.organization_id", org)
107+
.build()
108+
);
109+
181110
assertThat(
182-
err.getMessage(),
183-
is(
184-
"Cannot have global labels with tracing.agent prefix [organization_id=456] and"
185-
+ " telemetry.apm.agent prefix [deployment_id=123]"
111+
extracted,
112+
allOf(
113+
hasEntry("server_url", "https://myurl:443"),
114+
hasEntry("service_node_name", "instance-0000000001"),
115+
hasEntry(equalTo("global_labels"), not(endsWith(","))), // test that we have collapsed all global labels into one
116+
not(hasKey("global_labels.organization_id")) // tests that we strip out the top level label keys
186117
)
187118
);
119+
120+
List<String> labels = Arrays.stream(extracted.get("global_labels").split(",")).toList();
121+
assertThat(labels, hasSize(3));
122+
assertThat(labels, containsInAnyOrder("deployment_name=APM Tracing", "organization_id=" + org, "deployment_id=" + deploy));
123+
124+
// test replacing with underscores and skipping empty
125+
name = "APM=Tracing";
126+
deploy = "";
127+
org = ",456";
128+
extracted = APMJvmOptions.extractApmSettings(
129+
Settings.builder()
130+
.put(defaults)
131+
.put("telemetry.agent.global_labels.deployment_name", name)
132+
.put("telemetry.agent.global_labels.deployment_id", deploy)
133+
.put("telemetry.agent.global_labels.organization_id", org)
134+
.build()
135+
);
136+
labels = Arrays.stream(extracted.get("global_labels").split(",")).toList();
137+
assertThat(labels, hasSize(2));
138+
assertThat(labels, containsInAnyOrder("deployment_name=APM_Tracing", "organization_id=_456"));
188139
}
189140

190141
private Path makeFakeAgentJar() throws IOException {

‎docs/build.gradle‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -120,6 +120,8 @@ testClusters.matching { it.name == "yamlRestTest"}.configureEach {
120120
// TODO: remove this once cname is prepended to transport.publish_address by default in 8.0
121121
systemProperty 'es.transport.cname_in_publish_address', 'true'
122122

123+
systemProperty 'es.queryable_built_in_roles_enabled', 'false'
124+
123125
requiresFeature 'es.index_mode_feature_flag_registered', Version.fromString("8.0.0")
124126
requiresFeature 'es.failure_store_feature_flag_enabled', Version.fromString("8.12.0")
125127

‎docs/changelog/117504.yaml‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
pr: 117504
2+
summary: Fix NPE caused by race condition in async search when minimise round trips
3+
is true
4+
area: Search
5+
type: bug
6+
issues: []

0 commit comments

Comments
 (0)