Skip to content

Commit 6b07204

Browse files
fcofdezfcofdez
committed
Merge remote-tracking branch 'origin/main' into bloom-filter-stored-fields-format
2 parents 6550601 + 5e471b4 commit 6b07204

File tree

110 files changed

+2948
-2651
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

110 files changed

+2948
-2651
lines changed

‎build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/transport/TransportVersionGenerationFuncTest.groovy‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -513,4 +513,19 @@ class TransportVersionGenerationFuncTest extends AbstractTransportVersionFuncTes
513513
assertGenerateAndValidateSuccess(result)
514514
assertUpperBound("9.2", "existing_92,8123000")
515515
}
516+
517+
def "generation cannot run on release branch"() {
518+
given:
519+
file("myserver/build.gradle") << """
520+
tasks.named('generateTransportVersion') {
521+
currentUpperBoundName = '9.1'
522+
}
523+
"""
524+
525+
when:
526+
def result = runGenerateTask().buildAndFail()
527+
528+
then:
529+
assertGenerateFailure(result, "Transport version generation cannot run on release branches")
530+
}
516531
}

‎build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/ElasticsearchBuildCompletePlugin.java‎

Lines changed: 15 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -48,6 +48,7 @@
4848
import java.util.Arrays;
4949
import java.util.List;
5050
import java.util.Optional;
51+
import java.util.concurrent.TimeUnit;
5152

5253
import javax.inject.Inject;
5354

@@ -199,7 +200,11 @@ public void execute(BuildFinishedFlowAction.Parameters parameters) throws FileNo
199200
try {
200201
// we are very generious here, as the upload can take
201202
// a long time depending on its size
202-
pb.start().waitFor(30, java.util.concurrent.TimeUnit.MINUTES);
203+
long timeoutSec = calculateUploadWaitTimeoutSeconds(uploadFile);
204+
boolean completedInTime = pb.start().waitFor(timeoutSec, TimeUnit.SECONDS);
205+
if (completedInTime == false) {
206+
System.out.println("Timed out waiting for buildkite artifact upload after " + timeoutSec + " seconds");
207+
}
203208
} catch (InterruptedException e) {
204209
System.out.println("Failed to upload buildkite artifact " + e.getMessage());
205210
}
@@ -304,5 +309,14 @@ private static String calculateArchivePath(Path path, Path projectPath) {
304309
}
305310
return archivePath;
306311
}
312+
313+
private static long calculateUploadWaitTimeoutSeconds(File file) {
314+
long fileSizeBytes = file.length();
315+
long fileSizeMB = fileSizeBytes / (1024 * 1024);
316+
317+
// Allocate 4 seconds per MB (assumes ~250 KB/s upload speed)
318+
// with min 10 seconds and max 30 minutes
319+
return Math.max(10, Math.min(1800, fileSizeMB * 4));
320+
}
307321
}
308322
}

‎build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GenerateTransportVersionDefinitionTask.java‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -96,6 +96,12 @@ public abstract class GenerateTransportVersionDefinitionTask extends DefaultTask
9696
@TaskAction
9797
public void run() throws IOException {
9898
TransportVersionResourcesService resources = getResourceService().get();
99+
List<TransportVersionUpperBound> upstreamUpperBounds = resources.getUpperBoundsFromGitBase();
100+
boolean onReleaseBranch = resources.checkIfDefinitelyOnReleaseBranch(upstreamUpperBounds, getCurrentUpperBoundName().get());
101+
if (onReleaseBranch) {
102+
throw new IllegalArgumentException("Transport version generation cannot run on release branches");
103+
}
104+
99105
Set<String> referencedNames = TransportVersionReference.collectNames(getReferencesFiles());
100106
Set<String> changedDefinitionNames = resources.getChangedReferableDefinitionNames();
101107
String targetDefinitionName = getTargetDefinitionName(resources, referencedNames, changedDefinitionNames);
@@ -109,7 +115,7 @@ public void run() throws IOException {
109115
resetAllUpperBounds(resources, idsByBase);
110116
} else {
111117
getLogger().lifecycle("Generating transport version name: " + targetDefinitionName);
112-
List<TransportVersionUpperBound> upstreamUpperBounds = resources.getUpperBoundsFromGitBase();
118+
113119
Set<String> targetUpperBoundNames = getTargetUpperBoundNames(resources, upstreamUpperBounds, targetDefinitionName);
114120

115121
List<TransportVersionId> ids = updateUpperBounds(

‎build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionResourcesService.java‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@
2525
import java.nio.file.Files;
2626
import java.nio.file.Path;
2727
import java.util.ArrayList;
28+
import java.util.Collection;
2829
import java.util.Collections;
2930
import java.util.Comparator;
3031
import java.util.HashMap;
@@ -259,6 +260,20 @@ private Path getUpperBoundRelativePath(String name) {
259260
return UPPER_BOUNDS_DIR.resolve(name + ".csv");
260261
}
261262

263+
boolean checkIfDefinitelyOnReleaseBranch(Collection<TransportVersionUpperBound> upperBounds, String currentUpperBoundName) {
264+
// only want to look at definitions <= the current upper bound.
265+
// TODO: we should filter all of the upper bounds/definitions that are validated by this, not just in this method
266+
TransportVersionUpperBound currentUpperBound = upperBounds.stream()
267+
.filter(u -> u.name().equals(currentUpperBoundName))
268+
.findFirst()
269+
.orElse(null);
270+
if (currentUpperBound == null) {
271+
// since there is no current upper bound, we don't know if we are on a release branch
272+
return false;
273+
}
274+
return upperBounds.stream().anyMatch(u -> u.definitionId().complete() > currentUpperBound.definitionId().complete());
275+
}
276+
262277
private String getBaseRefName() {
263278
if (baseRefName.get() == null) {
264279
synchronized (baseRefName) {

‎build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/ValidateTransportVersionResourcesTask.java‎

Lines changed: 1 addition & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -84,7 +84,7 @@ public void validateTransportVersions() throws IOException {
8484
Map<Integer, List<IdAndDefinition>> idsByBase = resources.getIdsByBase();
8585
Map<String, TransportVersionUpperBound> upperBounds = resources.getUpperBounds();
8686
TransportVersionUpperBound currentUpperBound = upperBounds.get(getCurrentUpperBoundName().get());
87-
boolean onReleaseBranch = checkIfDefinitelyOnReleaseBranch(upperBounds);
87+
boolean onReleaseBranch = resources.checkIfDefinitelyOnReleaseBranch(upperBounds.values(), getCurrentUpperBoundName().get());
8888
boolean validateModifications = onReleaseBranch == false || getCI().get();
8989

9090
for (var definition : referableDefinitions.values()) {
@@ -330,15 +330,6 @@ private void validatePrimaryIds(
330330
);
331331
}
332332

333-
private boolean checkIfDefinitelyOnReleaseBranch(Map<String, TransportVersionUpperBound> upperBounds) {
334-
// only want to look at definitions <= the current upper bound.
335-
// TODO: we should filter all of the upper bounds/definitions that are validated by this, not just in this method
336-
String currentUpperBoundName = getCurrentUpperBoundName().get();
337-
TransportVersionUpperBound currentUpperBound = upperBounds.get(currentUpperBoundName);
338-
339-
return upperBounds.values().stream().anyMatch(u -> u.definitionId().complete() > currentUpperBound.definitionId().complete());
340-
}
341-
342333
private void throwDefinitionFailure(TransportVersionDefinition definition, String message) {
343334
Path relativePath = getResources().get().getDefinitionPath(definition);
344335
throw new VerificationException("Transport version definition file [" + relativePath + "] " + message);

‎build-tools/src/main/java/org/elasticsearch/gradle/testclusters/MockApmServer.java‎

Lines changed: 29 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -9,8 +9,11 @@
99

1010
package org.elasticsearch.gradle.testclusters;
1111

12+
import com.fasterxml.jackson.databind.JsonNode;
1213
import com.fasterxml.jackson.databind.ObjectMapper;
1314
import com.fasterxml.jackson.databind.node.ObjectNode;
15+
import com.fasterxml.jackson.databind.util.LRUMap;
16+
import com.fasterxml.jackson.databind.util.LookupCache;
1417
import com.sun.net.httpserver.HttpExchange;
1518
import com.sun.net.httpserver.HttpHandler;
1619
import com.sun.net.httpserver.HttpServer;
@@ -28,7 +31,9 @@
2831
import java.io.InputStreamReader;
2932
import java.io.OutputStream;
3033
import java.net.InetSocketAddress;
34+
import java.util.ArrayList;
3135
import java.util.Arrays;
36+
import java.util.List;
3237
import java.util.regex.Pattern;
3338
import java.util.stream.Collectors;
3439

@@ -48,6 +53,8 @@
4853
public class MockApmServer {
4954
private static final Logger logger = Logging.getLogger(MockApmServer.class);
5055
private static final org.slf4j.Logger log = LoggerFactory.getLogger(MockApmServer.class);
56+
private static final LookupCache<String, String> transactionCache = new LRUMap(16, 16);
57+
5158
private final Pattern metricFilter;
5259
private final Pattern transactionFilter;
5360
private final Pattern transactionExcludesFilter;
@@ -136,22 +143,28 @@ private void logFiltered(InputStream body) throws IOException {
136143
ObjectMapper mapper = new ObjectMapper();
137144
try (BufferedReader reader = new BufferedReader(new InputStreamReader(body))) {
138145
String line;
139-
String tier = null;
140-
String node = null;
146+
String nodeMetadata = null;
147+
148+
List<JsonNode> spans = new ArrayList<>();
141149

142150
while ((line = reader.readLine()) != null) {
143151
var jsonNode = mapper.readTree(line);
144152

145153
if (jsonNode.has("metadata")) {
146-
node = jsonNode.path("metadata").path("service").path("node").path("configured_name").asText(null);
147-
tier = jsonNode.path("metadata").path("labels").path("node_tier").asText(null);
154+
nodeMetadata = jsonNode.path("metadata").path("service").path("node").path("configured_name").asText(null);
155+
var tier = jsonNode.path("metadata").path("labels").path("node_tier").asText(null);
156+
nodeMetadata += tier != null ? "/" + tier : "";
157+
148158
} else if (transactionFilter != null && jsonNode.has("transaction")) {
149159
var transaction = jsonNode.get("transaction");
150160
var name = transaction.get("name").asText();
151161
if (transactionFilter.matcher(name).matches()
152162
&& (transactionExcludesFilter == null || transactionExcludesFilter.matcher(name).matches() == false)) {
153-
logger.lifecycle("Transaction [{}/{}]: {}", node, tier, transaction);
163+
transactionCache.put(transaction.get("id").asText(), name);
164+
logger.lifecycle("Transaction {} [{}]: {}", name, nodeMetadata, transaction);
154165
}
166+
} else if (jsonNode.has("span")) {
167+
spans.add(jsonNode.get("span")); // make sure to record all transactions first
155168
} else if (metricFilter != null && jsonNode.has("metricset")) {
156169
var metricset = jsonNode.get("metricset");
157170
var samples = (ObjectNode) metricset.get("samples");
@@ -161,10 +174,20 @@ private void logFiltered(InputStream body) throws IOException {
161174
}
162175
}
163176
if (samples.isEmpty() == false) {
164-
logger.lifecycle("Metricset [{}/{}]", node, tier, metricset);
177+
logger.lifecycle("Metricset [{}]: {}", nodeMetadata, metricset);
165178
}
166179
}
167180
}
181+
182+
// emit only spans for previously matched transactions using the transaction cache
183+
for (var span : spans) {
184+
var name = span.get("name").asText();
185+
var transactionId = span.get("transaction_id").asText();
186+
var transactionName = transactionCache.get(transactionId);
187+
if (transactionName != null) {
188+
logger.lifecycle("Span {} of {} [{}]: {}", name, transactionName, nodeMetadata, span);
189+
}
190+
}
168191
}
169192
}
170193
}

‎distribution/tools/plugin-cli/src/main/java/org/elasticsearch/plugins/cli/InstallPluginAction.java‎

Lines changed: 26 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -63,6 +63,7 @@
6363
import java.nio.file.FileVisitResult;
6464
import java.nio.file.Files;
6565
import java.nio.file.Path;
66+
import java.nio.file.Paths;
6667
import java.nio.file.SimpleFileVisitor;
6768
import java.nio.file.StandardCopyOption;
6869
import java.nio.file.attribute.BasicFileAttributes;
@@ -329,10 +330,31 @@ private Path download(InstallablePlugin plugin, Path tmpDir) throws Exception {
329330
}
330331
throw new UserException(ExitCodes.USAGE, msg);
331332
}
333+
334+
verifyLocationNotInPluginsDirectory(pluginLocation);
335+
332336
terminal.println(logPrefix + "Downloading " + URLDecoder.decode(pluginLocation, StandardCharsets.UTF_8));
333337
return downloadZip(pluginLocation, tmpDir);
334338
}
335339

340+
@SuppressForbidden(reason = "Need to use Paths#get")
341+
private void verifyLocationNotInPluginsDirectory(String pluginLocation) throws URISyntaxException, IOException, UserException {
342+
if (pluginLocation == null) {
343+
return;
344+
}
345+
URI uri = new URI(pluginLocation);
346+
if ("file".equalsIgnoreCase(uri.getScheme())) {
347+
Path pluginRealPath = Paths.get(uri).toRealPath();
348+
Path pluginsDirectory = env.pluginsDir().toRealPath();
349+
if (pluginRealPath.startsWith(pluginsDirectory)) {
350+
throw new UserException(
351+
ExitCodes.USAGE,
352+
"Installation of plugin in location [" + pluginLocation + "] from inside the plugins directory is not permitted."
353+
);
354+
}
355+
}
356+
}
357+
336358
@SuppressForbidden(reason = "Need to use PathUtils#get")
337359
private Path getPluginArchivePath(String pluginId, String pluginArchiveDir) throws UserException {
338360
final Path path = PathUtils.get(pluginArchiveDir);
@@ -462,9 +484,9 @@ private static List<String> checkMisspelledPlugin(String pluginId) {
462484
/** Downloads a zip from the url, into a temp file under the given temp dir. */
463485
// pkg private for tests
464486
@SuppressForbidden(reason = "We use getInputStream to download plugins")
465-
Path downloadZip(String urlString, Path tmpDir) throws IOException {
487+
Path downloadZip(String urlString, Path tmpDir) throws IOException, URISyntaxException {
466488
terminal.println(VERBOSE, "Retrieving zip from " + urlString);
467-
URL url = new URL(urlString);
489+
URL url = new URI(urlString).toURL();
468490
Path zip = Files.createTempFile(tmpDir, null, ".zip");
469491
URLConnection urlConnection = this.proxy == null ? url.openConnection() : url.openConnection(this.proxy);
470492
urlConnection.addRequestProperty("User-Agent", "elasticsearch-plugin-installer");
@@ -548,9 +570,10 @@ private InputStream urlOpenStream(final URL url) throws IOException {
548570
* @throws IOException if an I/O exception occurs download or reading files and resources
549571
* @throws PGPException if an exception occurs verifying the downloaded ZIP signature
550572
* @throws UserException if checksum validation fails
573+
* @throws URISyntaxException is the url is invalid
551574
*/
552575
private Path downloadAndValidate(final String urlString, final Path tmpDir, final boolean officialPlugin) throws IOException,
553-
PGPException, UserException {
576+
PGPException, UserException, URISyntaxException {
554577
Path zip = downloadZip(urlString, tmpDir);
555578
pathsToDeleteOnShutdown.add(zip);
556579
String checksumUrlString = urlString + ".sha512";

‎distribution/tools/plugin-cli/src/test/java/org/elasticsearch/plugins/cli/InstallPluginActionTests.java‎

Lines changed: 22 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -66,18 +66,18 @@
6666
import java.io.BufferedReader;
6767
import java.io.ByteArrayInputStream;
6868
import java.io.ByteArrayOutputStream;
69-
import java.io.FileNotFoundException;
7069
import java.io.IOException;
7170
import java.io.InputStream;
7271
import java.io.StringReader;
73-
import java.net.MalformedURLException;
7472
import java.net.URI;
73+
import java.net.URISyntaxException;
7574
import java.net.URL;
7675
import java.nio.charset.StandardCharsets;
7776
import java.nio.file.DirectoryStream;
7877
import java.nio.file.FileAlreadyExistsException;
7978
import java.nio.file.FileSystem;
8079
import java.nio.file.Files;
80+
import java.nio.file.NoSuchFileException;
8181
import java.nio.file.Path;
8282
import java.nio.file.StandardCopyOption;
8383
import java.nio.file.attribute.GroupPrincipal;
@@ -552,8 +552,8 @@ public void testTransaction() throws Exception {
552552
pluginZip.getId() + "-does-not-exist",
553553
pluginZip.getLocation() + "-does-not-exist"
554554
);
555-
final FileNotFoundException e = expectThrows(
556-
FileNotFoundException.class,
555+
final NoSuchFileException e = expectThrows(
556+
NoSuchFileException.class,
557557
() -> installPlugins(List.of(pluginZip, nonexistentPluginZip), env.v1())
558558
);
559559
assertThat(e.getMessage(), containsString("does-not-exist"));
@@ -586,11 +586,27 @@ public void testSpaceInUrl() throws Exception {
586586
assertPlugin("fake", pluginDir, env.v2());
587587
}
588588

589+
public void testCannotInstallFromInsidePluginsDirectory() throws Exception {
590+
InstallablePlugin pluginZip = createPluginZip("fake", pluginDir);
591+
Path pluginZipInsidePlugins = env.v2().pluginsDir().resolve("fake.zip");
592+
try (InputStream in = FileSystemUtils.openFileURLStream(new URL(pluginZip.getLocation()))) {
593+
Files.copy(in, pluginZipInsidePlugins, StandardCopyOption.REPLACE_EXISTING);
594+
}
595+
String location = pluginZipInsidePlugins.toUri().toURL().toString();
596+
assumeTrue("requires file URL scheme", location.startsWith("file:"));
597+
InstallablePlugin modifiedPlugin = new InstallablePlugin("fake", location);
598+
UserException e = expectThrows(UserException.class, () -> installPlugin(modifiedPlugin));
599+
assertThat(
600+
e.getMessage(),
601+
startsWith("Installation of plugin in location [" + location + "] from inside the plugins directory is not permitted.")
602+
);
603+
}
604+
589605
public void testMalformedUrlNotMaven() {
590606
// has two colons, so it appears similar to maven coordinates
591607
InstallablePlugin plugin = new InstallablePlugin("fake", "://host:1234");
592-
MalformedURLException e = expectThrows(MalformedURLException.class, () -> installPlugin(plugin));
593-
assertThat(e.getMessage(), containsString("no protocol"));
608+
URISyntaxException e = expectThrows(URISyntaxException.class, () -> installPlugin(plugin));
609+
assertThat(e.getMessage(), containsString("Expected scheme name"));
594610
}
595611

596612
public void testFileNotMaven() {

‎distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/APMJvmOptions.java‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -81,7 +81,10 @@ class APMJvmOptions {
8181
"metrics_interval", "120s",
8282
"breakdown_metrics", "false",
8383
"central_config", "false",
84-
"transaction_sample_rate", "0.2"
84+
"transaction_sample_rate", "0.2",
85+
// Don't collect stacktraces for spans, typically these are of little use as
86+
// always pointing to APMTracer.stopTrace invoked from TaskManager
87+
"stack_trace_limit", "0"
8588
);
8689
// end::noformat
8790

‎docs/changelog/135231.yaml‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
pr: 135231
2+
summary: Improve retrying PIT contexts for read-only indices
3+
area: Search
4+
type: enhancement
5+
issues: []

0 commit comments

Comments
 (0)