Skip to content

Commit 152bdc0

Browse files
rjernstrjernst
authored
Merge branch '8.19' into transport/resolve_conflict_without_compile_8.19
2 parents 6229e58 + 1bc004c commit 152bdc0

File tree

62 files changed

+771
-169
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

62 files changed

+771
-169
lines changed

‎distribution/docker/src/docker/dockerfiles/cloud_ess_fips/Dockerfile‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,7 @@
2525
# Extract Elasticsearch artifact
2626
################################################################################
2727
28-
FROM docker.elastic.co/wolfi/chainguard-base-fips:latest@sha256:bf732027be3f7f5ecc5fd5334cf5d12c67d5f2a71ebf4230309d0e34b0cb47ad AS builder
28+
FROM docker.elastic.co/wolfi/chainguard-base-fips:latest@sha256:11d66ae7ec7ca1d5a7289750bdd96eb3d8eb592e5b7377f1fc8e4fb556fa7383 AS builder
2929
3030
# Install required packages to extract the Elasticsearch distribution
3131
RUN <%= retry.loop(package_manager, "export DEBIAN_FRONTEND=noninteractive && ${package_manager} update && ${package_manager} update && ${package_manager} add --no-cache curl") %>
@@ -104,7 +104,7 @@ WORKDIR /usr/share/elasticsearch/config
104104
# Add entrypoint
105105
################################################################################
106106

107-
FROM docker.elastic.co/wolfi/chainguard-base-fips:latest@sha256:bf732027be3f7f5ecc5fd5334cf5d12c67d5f2a71ebf4230309d0e34b0cb47ad
107+
FROM docker.elastic.co/wolfi/chainguard-base-fips:latest@sha256:11d66ae7ec7ca1d5a7289750bdd96eb3d8eb592e5b7377f1fc8e4fb556fa7383
108108

109109
RUN <%= retry.loop(package_manager,
110110
"export DEBIAN_FRONTEND=noninteractive && \n" +

‎distribution/docker/src/docker/dockerfiles/wolfi/Dockerfile‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,7 @@
2525
# Extract Elasticsearch artifact
2626
################################################################################
2727
28-
FROM docker.elastic.co/wolfi/chainguard-base:latest@sha256:a42fd0f8865a04b8ab4ff12e9ecbba681ebae4112b9e7e9e34ce9b08ebfa650d AS builder
28+
FROM docker.elastic.co/wolfi/chainguard-base:latest@sha256:b5a03b6a754fa2f9a29e44e316a6c4df1f606cd8a3cd8810ce3d6a3a3134428b AS builder
2929
3030
# Install required packages to extract the Elasticsearch distribution
3131
RUN <%= retry.loop(package_manager, "export DEBIAN_FRONTEND=noninteractive && ${package_manager} update && ${package_manager} update && ${package_manager} add --no-cache curl") %>
@@ -68,7 +68,7 @@ RUN sed -i -e 's/ES_DISTRIBUTION_TYPE=tar/ES_DISTRIBUTION_TYPE=docker/' bin/elas
6868
# Add entrypoint
6969
################################################################################
7070

71-
FROM docker.elastic.co/wolfi/chainguard-base:latest@sha256:a42fd0f8865a04b8ab4ff12e9ecbba681ebae4112b9e7e9e34ce9b08ebfa650d
71+
FROM docker.elastic.co/wolfi/chainguard-base:latest@sha256:b5a03b6a754fa2f9a29e44e316a6c4df1f606cd8a3cd8810ce3d6a3a3134428b
7272

7373
RUN <%= retry.loop(package_manager,
7474
"export DEBIAN_FRONTEND=noninteractive && \n" +

‎distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/JvmArgumentParsingSystemMemoryInfo.java‎

Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,38 @@
1+
/*
2+
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
3+
* or more contributor license agreements. Licensed under the "Elastic License
4+
* 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
5+
* Public License v 1"; you may not use this file except in compliance with, at
6+
* your election, the "Elastic License 2.0", the "GNU Affero General Public
7+
* License v3.0 only", or the "Server Side Public License, v 1".
8+
*/
9+
10+
package org.elasticsearch.server.cli;
11+
12+
import java.util.List;
13+
14+
public abstract class JvmArgumentParsingSystemMemoryInfo implements SystemMemoryInfo {
15+
private final List<String> userDefinedJvmOptions;
16+
17+
public JvmArgumentParsingSystemMemoryInfo(List<String> userDefinedJvmOptions) {
18+
this.userDefinedJvmOptions = userDefinedJvmOptions;
19+
}
20+
21+
protected long getBytesFromSystemProperty(String systemProperty, long defaultValue) {
22+
return userDefinedJvmOptions.stream()
23+
.filter(option -> option.startsWith("-D" + systemProperty + "="))
24+
.map(totalMemoryOverheadBytesOption -> {
25+
try {
26+
long bytes = Long.parseLong(totalMemoryOverheadBytesOption.split("=", 2)[1]);
27+
if (bytes < 0) {
28+
throw new IllegalArgumentException("Negative bytes size specified in [" + totalMemoryOverheadBytesOption + "]");
29+
}
30+
return bytes;
31+
} catch (NumberFormatException e) {
32+
throw new IllegalArgumentException("Unable to parse number of bytes from [" + totalMemoryOverheadBytesOption + "]", e);
33+
}
34+
})
35+
.reduce((previous, current) -> current) // this is effectively findLast(), so that ES_JAVA_OPTS overrides jvm.options
36+
.orElse(defaultValue);
37+
}
38+
}

‎distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/JvmOptionsParser.java‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -142,7 +142,10 @@ private List<String> jvmOptions(
142142
}
143143

144144
final List<String> substitutedJvmOptions = substitutePlaceholders(jvmOptions, Collections.unmodifiableMap(substitutions));
145-
final SystemMemoryInfo memoryInfo = new OverridableSystemMemoryInfo(substitutedJvmOptions, new DefaultSystemMemoryInfo());
145+
final SystemMemoryInfo memoryInfo = new OverheadSystemMemoryInfo(
146+
substitutedJvmOptions,
147+
new OverridableSystemMemoryInfo(substitutedJvmOptions, new DefaultSystemMemoryInfo())
148+
);
146149
substitutedJvmOptions.addAll(machineDependentHeap.determineHeapSettings(args.nodeSettings(), memoryInfo, substitutedJvmOptions));
147150
final List<String> ergonomicJvmOptions = JvmErgonomics.choose(substitutedJvmOptions, args.nodeSettings());
148151
final List<String> systemJvmOptions = SystemJvmOptions.systemJvmOptions(args.nodeSettings(), cliSysprops);

‎distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/OverheadSystemMemoryInfo.java‎

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,35 @@
1+
/*
2+
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
3+
* or more contributor license agreements. Licensed under the "Elastic License
4+
* 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
5+
* Public License v 1"; you may not use this file except in compliance with, at
6+
* your election, the "Elastic License 2.0", the "GNU Affero General Public
7+
* License v3.0 only", or the "Server Side Public License, v 1".
8+
*/
9+
10+
package org.elasticsearch.server.cli;
11+
12+
import java.util.List;
13+
14+
/**
15+
* A {@link SystemMemoryInfo} implementation that reduces the reported available system memory by a set amount. This is intended to account
16+
* for overhead cost of other bundled Elasticsearch processes, such as the CLI tool launcher. By default, this is
17+
* {@link org.elasticsearch.server.cli.OverheadSystemMemoryInfo#SERVER_CLI_OVERHEAD } but can be overridden via the
18+
* {@code es.total_memory_overhead_bytes} system property.
19+
*/
20+
public class OverheadSystemMemoryInfo extends JvmArgumentParsingSystemMemoryInfo {
21+
static final long SERVER_CLI_OVERHEAD = 100 * 1024L * 1024L;
22+
23+
private final SystemMemoryInfo delegate;
24+
25+
public OverheadSystemMemoryInfo(List<String> userDefinedJvmOptions, SystemMemoryInfo delegate) {
26+
super(userDefinedJvmOptions);
27+
this.delegate = delegate;
28+
}
29+
30+
@Override
31+
public long availableSystemMemory() {
32+
long overheadBytes = getBytesFromSystemProperty("es.total_memory_overhead_bytes", SERVER_CLI_OVERHEAD);
33+
return delegate.availableSystemMemory() - overheadBytes;
34+
}
35+
}

‎distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/OverridableSystemMemoryInfo.java‎

Lines changed: 3 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -17,33 +17,17 @@
1717
* has been specified using the {@code es.total_memory_bytes} system property, or
1818
* else returns the value provided by a fallback provider.
1919
*/
20-
public final class OverridableSystemMemoryInfo implements SystemMemoryInfo {
20+
public final class OverridableSystemMemoryInfo extends JvmArgumentParsingSystemMemoryInfo {
2121

22-
private final List<String> userDefinedJvmOptions;
2322
private final SystemMemoryInfo fallbackSystemMemoryInfo;
2423

2524
public OverridableSystemMemoryInfo(final List<String> userDefinedJvmOptions, SystemMemoryInfo fallbackSystemMemoryInfo) {
26-
this.userDefinedJvmOptions = Objects.requireNonNull(userDefinedJvmOptions);
25+
super(userDefinedJvmOptions);
2726
this.fallbackSystemMemoryInfo = Objects.requireNonNull(fallbackSystemMemoryInfo);
2827
}
2928

3029
@Override
3130
public long availableSystemMemory() {
32-
33-
return userDefinedJvmOptions.stream()
34-
.filter(option -> option.startsWith("-Des.total_memory_bytes="))
35-
.map(totalMemoryBytesOption -> {
36-
try {
37-
long bytes = Long.parseLong(totalMemoryBytesOption.split("=", 2)[1]);
38-
if (bytes < 0) {
39-
throw new IllegalArgumentException("Negative memory size specified in [" + totalMemoryBytesOption + "]");
40-
}
41-
return bytes;
42-
} catch (NumberFormatException e) {
43-
throw new IllegalArgumentException("Unable to parse number of bytes from [" + totalMemoryBytesOption + "]", e);
44-
}
45-
})
46-
.reduce((previous, current) -> current) // this is effectively findLast(), so that ES_JAVA_OPTS overrides jvm.options
47-
.orElse(fallbackSystemMemoryInfo.availableSystemMemory());
31+
return getBytesFromSystemProperty("es.total_memory_bytes", fallbackSystemMemoryInfo.availableSystemMemory());
4832
}
4933
}

‎distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/ServerCli.java‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -118,6 +118,8 @@ public void execute(Terminal terminal, OptionSet options, Environment env, Proce
118118
return;
119119
}
120120

121+
// Call the GC to try and free up as much heap as we can since we don't intend to do much if any more allocation after this
122+
System.gc();
121123
// we are running in the foreground, so wait for the server to exit
122124
int exitCode = server.waitFor();
123125
onExit(exitCode);

‎distribution/tools/server-cli/src/test/java/org/elasticsearch/server/cli/OverheadSystemMemoryInfoTests.java‎

Lines changed: 86 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,86 @@
1+
/*
2+
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
3+
* or more contributor license agreements. Licensed under the "Elastic License
4+
* 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
5+
* Public License v 1"; you may not use this file except in compliance with, at
6+
* your election, the "Elastic License 2.0", the "GNU Affero General Public
7+
* License v3.0 only", or the "Server Side Public License, v 1".
8+
*/
9+
10+
package org.elasticsearch.server.cli;
11+
12+
import org.elasticsearch.test.ESTestCase;
13+
14+
import java.util.List;
15+
16+
import static org.elasticsearch.server.cli.OverheadSystemMemoryInfo.SERVER_CLI_OVERHEAD;
17+
import static org.hamcrest.Matchers.is;
18+
19+
public class OverheadSystemMemoryInfoTests extends ESTestCase {
20+
21+
private static final long TRUE_SYSTEM_MEMORY = 1024 * 1024 * 1024L;
22+
23+
public void testNoOptions() {
24+
final SystemMemoryInfo memoryInfo = new OverheadSystemMemoryInfo(List.of(), delegateSystemMemoryInfo());
25+
assertThat(memoryInfo.availableSystemMemory(), is(TRUE_SYSTEM_MEMORY - SERVER_CLI_OVERHEAD));
26+
}
27+
28+
public void testNoOverrides() {
29+
final SystemMemoryInfo memoryInfo = new OverheadSystemMemoryInfo(List.of("-Da=b", "-Dx=y"), delegateSystemMemoryInfo());
30+
assertThat(memoryInfo.availableSystemMemory(), is(TRUE_SYSTEM_MEMORY - SERVER_CLI_OVERHEAD));
31+
}
32+
33+
public void testValidSingleOverride() {
34+
final SystemMemoryInfo memoryInfo = new OverheadSystemMemoryInfo(
35+
List.of("-Des.total_memory_overhead_bytes=50000"),
36+
delegateSystemMemoryInfo()
37+
);
38+
assertThat(memoryInfo.availableSystemMemory(), is(TRUE_SYSTEM_MEMORY - 50000));
39+
}
40+
41+
public void testValidOverrideInList() {
42+
final SystemMemoryInfo memoryInfo = new OverheadSystemMemoryInfo(
43+
List.of("-Da=b", "-Des.total_memory_overhead_bytes=50000", "-Dx=y"),
44+
delegateSystemMemoryInfo()
45+
);
46+
assertThat(memoryInfo.availableSystemMemory(), is(TRUE_SYSTEM_MEMORY - 50000));
47+
}
48+
49+
public void testMultipleValidOverridesInList() {
50+
final SystemMemoryInfo memoryInfo = new OverheadSystemMemoryInfo(
51+
List.of("-Des.total_memory_overhead_bytes=50000", "-Da=b", "-Des.total_memory_overhead_bytes=100000", "-Dx=y"),
52+
delegateSystemMemoryInfo()
53+
);
54+
assertThat(memoryInfo.availableSystemMemory(), is(TRUE_SYSTEM_MEMORY - 100000));
55+
}
56+
57+
public void testNegativeOverride() {
58+
final SystemMemoryInfo memoryInfo = new OverheadSystemMemoryInfo(
59+
List.of("-Da=b", "-Des.total_memory_overhead_bytes=-123", "-Dx=y"),
60+
delegateSystemMemoryInfo()
61+
);
62+
try {
63+
memoryInfo.availableSystemMemory();
64+
fail("expected to fail");
65+
} catch (IllegalArgumentException e) {
66+
assertThat(e.getMessage(), is("Negative bytes size specified in [-Des.total_memory_overhead_bytes=-123]"));
67+
}
68+
}
69+
70+
public void testUnparsableOverride() {
71+
final SystemMemoryInfo memoryInfo = new OverheadSystemMemoryInfo(
72+
List.of("-Da=b", "-Des.total_memory_overhead_bytes=invalid", "-Dx=y"),
73+
delegateSystemMemoryInfo()
74+
);
75+
try {
76+
memoryInfo.availableSystemMemory();
77+
fail("expected to fail");
78+
} catch (IllegalArgumentException e) {
79+
assertThat(e.getMessage(), is("Unable to parse number of bytes from [-Des.total_memory_overhead_bytes=invalid]"));
80+
}
81+
}
82+
83+
private static SystemMemoryInfo delegateSystemMemoryInfo() {
84+
return () -> TRUE_SYSTEM_MEMORY;
85+
}
86+
}

‎distribution/tools/server-cli/src/test/java/org/elasticsearch/server/cli/OverridableSystemMemoryInfoTests.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -64,7 +64,7 @@ public void testNegativeOverride() {
6464
memoryInfo.availableSystemMemory();
6565
fail("expected to fail");
6666
} catch (IllegalArgumentException e) {
67-
assertThat(e.getMessage(), is("Negative memory size specified in [-Des.total_memory_bytes=-123]"));
67+
assertThat(e.getMessage(), is("Negative bytes size specified in [-Des.total_memory_bytes=-123]"));
6868
}
6969
}
7070

‎docs/changelog/141185.yaml‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
pr: 141185
2+
summary: Add `max_batch_size` setting to EIS dense and sparse service settings
3+
area: Inference
4+
type: enhancement
5+
issues: []

0 commit comments

Comments
 (0)