Merge branch 'main' into cert-audit-log-IT

Author: gmjehovich

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/ElasticsearchBuildCompletePlugin.java

@@ -48,6 +48,7 @@
 import java.util.Arrays;
 import java.util.List;
 import java.util.Optional;
+import java.util.concurrent.TimeUnit;
 
 import javax.inject.Inject;
 
@@ -199,7 +200,11 @@ public void execute(BuildFinishedFlowAction.Parameters parameters) throws FileNo
                     try {
                         // we are very generious here, as the upload can take
                         // a long time depending on its size
-                        pb.start().waitFor(30, java.util.concurrent.TimeUnit.MINUTES);
+                        long timeoutSec = calculateUploadWaitTimeoutSeconds(uploadFile);
+                        boolean completedInTime = pb.start().waitFor(timeoutSec, TimeUnit.SECONDS);
+                        if (completedInTime == false) {
+                            System.out.println("Timed out waiting for buildkite artifact upload after " + timeoutSec + " seconds");
+                        }
                     } catch (InterruptedException e) {
                         System.out.println("Failed to upload buildkite artifact " + e.getMessage());
                     }
@@ -304,5 +309,14 @@ private static String calculateArchivePath(Path path, Path projectPath) {
             }
             return archivePath;
         }
+
+        private static long calculateUploadWaitTimeoutSeconds(File file) {
+            long fileSizeBytes = file.length();
+            long fileSizeMB = fileSizeBytes / (1024 * 1024);
+
+            // Allocate 4 seconds per MB (assumes ~250 KB/s upload speed)
+            // with min 10 seconds and max 30 minutes
+            return Math.max(10, Math.min(1800, fileSizeMB * 4));
+        }
     }
 }

build-tools/src/main/java/org/elasticsearch/gradle/testclusters/MockApmServer.java

@@ -9,8 +9,11 @@
 
 package org.elasticsearch.gradle.testclusters;
 
+import com.fasterxml.jackson.databind.JsonNode;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.fasterxml.jackson.databind.node.ObjectNode;
+import com.fasterxml.jackson.databind.util.LRUMap;
+import com.fasterxml.jackson.databind.util.LookupCache;
 import com.sun.net.httpserver.HttpExchange;
 import com.sun.net.httpserver.HttpHandler;
 import com.sun.net.httpserver.HttpServer;
@@ -28,7 +31,9 @@
 import java.io.InputStreamReader;
 import java.io.OutputStream;
 import java.net.InetSocketAddress;
+import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.List;
 import java.util.regex.Pattern;
 import java.util.stream.Collectors;
 
@@ -48,6 +53,8 @@
 public class MockApmServer {
     private static final Logger logger = Logging.getLogger(MockApmServer.class);
     private static final org.slf4j.Logger log = LoggerFactory.getLogger(MockApmServer.class);
+    private static final LookupCache<String, String> transactionCache = new LRUMap(16, 16);
+
     private final Pattern metricFilter;
     private final Pattern transactionFilter;
     private final Pattern transactionExcludesFilter;
@@ -136,22 +143,28 @@ private void logFiltered(InputStream body) throws IOException {
             ObjectMapper mapper = new ObjectMapper();
             try (BufferedReader reader = new BufferedReader(new InputStreamReader(body))) {
                 String line;
-                String tier = null;
-                String node = null;
+                String nodeMetadata = null;
+
+                List<JsonNode> spans = new ArrayList<>();
 
                 while ((line = reader.readLine()) != null) {
                     var jsonNode = mapper.readTree(line);
 
                     if (jsonNode.has("metadata")) {
-                        node = jsonNode.path("metadata").path("service").path("node").path("configured_name").asText(null);
-                        tier = jsonNode.path("metadata").path("labels").path("node_tier").asText(null);
+                        nodeMetadata = jsonNode.path("metadata").path("service").path("node").path("configured_name").asText(null);
+                        var tier = jsonNode.path("metadata").path("labels").path("node_tier").asText(null);
+                        nodeMetadata += tier != null ? "/" + tier : "";
+
                     } else if (transactionFilter != null && jsonNode.has("transaction")) {
                         var transaction = jsonNode.get("transaction");
                         var name = transaction.get("name").asText();
                         if (transactionFilter.matcher(name).matches()
                             && (transactionExcludesFilter == null || transactionExcludesFilter.matcher(name).matches() == false)) {
-                            logger.lifecycle("Transaction [{}/{}]: {}", node, tier, transaction);
+                            transactionCache.put(transaction.get("id").asText(), name);
+                            logger.lifecycle("Transaction {} [{}]: {}", name, nodeMetadata, transaction);
                         }
+                    } else if (jsonNode.has("span")) {
+                        spans.add(jsonNode.get("span")); // make sure to record all transactions first
                     } else if (metricFilter != null && jsonNode.has("metricset")) {
                         var metricset = jsonNode.get("metricset");
                         var samples = (ObjectNode) metricset.get("samples");
@@ -161,10 +174,20 @@ private void logFiltered(InputStream body) throws IOException {
                             }
                         }
                         if (samples.isEmpty() == false) {
-                            logger.lifecycle("Metricset [{}/{}]", node, tier, metricset);
+                            logger.lifecycle("Metricset [{}]: {}", nodeMetadata, metricset);
                         }
                     }
                 }
+
+                // emit only spans for previously matched transactions using the transaction cache
+                for (var span : spans) {
+                    var name = span.get("name").asText();
+                    var transactionId = span.get("transaction_id").asText();
+                    var transactionName = transactionCache.get(transactionId);
+                    if (transactionName != null) {
+                        logger.lifecycle("Span {} of {} [{}]: {}", name, transactionName, nodeMetadata, span);
+                    }
+                }
             }
         }
     }

distribution/tools/plugin-cli/src/main/java/org/elasticsearch/plugins/cli/InstallPluginAction.java

@@ -63,6 +63,7 @@
 import java.nio.file.FileVisitResult;
 import java.nio.file.Files;
 import java.nio.file.Path;
+import java.nio.file.Paths;
 import java.nio.file.SimpleFileVisitor;
 import java.nio.file.StandardCopyOption;
 import java.nio.file.attribute.BasicFileAttributes;
@@ -329,10 +330,31 @@ private Path download(InstallablePlugin plugin, Path tmpDir) throws Exception {
             }
             throw new UserException(ExitCodes.USAGE, msg);
         }
+
+        verifyLocationNotInPluginsDirectory(pluginLocation);
+
         terminal.println(logPrefix + "Downloading " + URLDecoder.decode(pluginLocation, StandardCharsets.UTF_8));
         return downloadZip(pluginLocation, tmpDir);
     }
 
+    @SuppressForbidden(reason = "Need to use Paths#get")
+    private void verifyLocationNotInPluginsDirectory(String pluginLocation) throws URISyntaxException, IOException, UserException {
+        if (pluginLocation == null) {
+            return;
+        }
+        URI uri = new URI(pluginLocation);
+        if ("file".equalsIgnoreCase(uri.getScheme())) {
+            Path pluginRealPath = Paths.get(uri).toRealPath();
+            Path pluginsDirectory = env.pluginsDir().toRealPath();
+            if (pluginRealPath.startsWith(pluginsDirectory)) {
+                throw new UserException(
+                    ExitCodes.USAGE,
+                    "Installation of plugin in location [" + pluginLocation + "] from inside the plugins directory is not permitted."
+                );
+            }
+        }
+    }
+
     @SuppressForbidden(reason = "Need to use PathUtils#get")
     private Path getPluginArchivePath(String pluginId, String pluginArchiveDir) throws UserException {
         final Path path = PathUtils.get(pluginArchiveDir);
@@ -462,9 +484,9 @@ private static List<String> checkMisspelledPlugin(String pluginId) {
     /** Downloads a zip from the url, into a temp file under the given temp dir. */
     // pkg private for tests
     @SuppressForbidden(reason = "We use getInputStream to download plugins")
-    Path downloadZip(String urlString, Path tmpDir) throws IOException {
+    Path downloadZip(String urlString, Path tmpDir) throws IOException, URISyntaxException {
         terminal.println(VERBOSE, "Retrieving zip from " + urlString);
-        URL url = new URL(urlString);
+        URL url = new URI(urlString).toURL();
         Path zip = Files.createTempFile(tmpDir, null, ".zip");
         URLConnection urlConnection = this.proxy == null ? url.openConnection() : url.openConnection(this.proxy);
         urlConnection.addRequestProperty("User-Agent", "elasticsearch-plugin-installer");
@@ -548,9 +570,10 @@ private InputStream urlOpenStream(final URL url) throws IOException {
      * @throws IOException   if an I/O exception occurs download or reading files and resources
      * @throws PGPException  if an exception occurs verifying the downloaded ZIP signature
      * @throws UserException if checksum validation fails
+     * @throws URISyntaxException is the url is invalid
      */
     private Path downloadAndValidate(final String urlString, final Path tmpDir, final boolean officialPlugin) throws IOException,
-        PGPException, UserException {
+        PGPException, UserException, URISyntaxException {
         Path zip = downloadZip(urlString, tmpDir);
         pathsToDeleteOnShutdown.add(zip);
         String checksumUrlString = urlString + ".sha512";

distribution/tools/plugin-cli/src/test/java/org/elasticsearch/plugins/cli/InstallPluginActionTests.java

@@ -66,18 +66,18 @@
 import java.io.BufferedReader;
 import java.io.ByteArrayInputStream;
 import java.io.ByteArrayOutputStream;
-import java.io.FileNotFoundException;
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.StringReader;
-import java.net.MalformedURLException;
 import java.net.URI;
+import java.net.URISyntaxException;
 import java.net.URL;
 import java.nio.charset.StandardCharsets;
 import java.nio.file.DirectoryStream;
 import java.nio.file.FileAlreadyExistsException;
 import java.nio.file.FileSystem;
 import java.nio.file.Files;
+import java.nio.file.NoSuchFileException;
 import java.nio.file.Path;
 import java.nio.file.StandardCopyOption;
 import java.nio.file.attribute.GroupPrincipal;
@@ -552,8 +552,8 @@ public void testTransaction() throws Exception {
             pluginZip.getId() + "-does-not-exist",
             pluginZip.getLocation() + "-does-not-exist"
         );
-        final FileNotFoundException e = expectThrows(
-            FileNotFoundException.class,
+        final NoSuchFileException e = expectThrows(
+            NoSuchFileException.class,
             () -> installPlugins(List.of(pluginZip, nonexistentPluginZip), env.v1())
         );
         assertThat(e.getMessage(), containsString("does-not-exist"));
@@ -586,11 +586,27 @@ public void testSpaceInUrl() throws Exception {
         assertPlugin("fake", pluginDir, env.v2());
     }
 
+    public void testCannotInstallFromInsidePluginsDirectory() throws Exception {
+        InstallablePlugin pluginZip = createPluginZip("fake", pluginDir);
+        Path pluginZipInsidePlugins = env.v2().pluginsDir().resolve("fake.zip");
+        try (InputStream in = FileSystemUtils.openFileURLStream(new URL(pluginZip.getLocation()))) {
+            Files.copy(in, pluginZipInsidePlugins, StandardCopyOption.REPLACE_EXISTING);
+        }
+        String location = pluginZipInsidePlugins.toUri().toURL().toString();
+        assumeTrue("requires file URL scheme", location.startsWith("file:"));
+        InstallablePlugin modifiedPlugin = new InstallablePlugin("fake", location);
+        UserException e = expectThrows(UserException.class, () -> installPlugin(modifiedPlugin));
+        assertThat(
+            e.getMessage(),
+            startsWith("Installation of plugin in location [" + location + "] from inside the plugins directory is not permitted.")
+        );
+    }
+
     public void testMalformedUrlNotMaven() {
         // has two colons, so it appears similar to maven coordinates
         InstallablePlugin plugin = new InstallablePlugin("fake", "://host:1234");
-        MalformedURLException e = expectThrows(MalformedURLException.class, () -> installPlugin(plugin));
-        assertThat(e.getMessage(), containsString("no protocol"));
+        URISyntaxException e = expectThrows(URISyntaxException.class, () -> installPlugin(plugin));
+        assertThat(e.getMessage(), containsString("Expected scheme name"));
     }
 
     public void testFileNotMaven() {

distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/APMJvmOptions.java

@@ -81,7 +81,10 @@ class APMJvmOptions {
         "metrics_interval", "120s",
         "breakdown_metrics", "false",
         "central_config", "false",
-        "transaction_sample_rate", "0.2"
+        "transaction_sample_rate", "0.2",
+        // Don't collect stacktraces for spans, typically these are of little use as
+        // always pointing to APMTracer.stopTrace invoked from TaskManager
+        "stack_trace_limit", "0"
         );
     // end::noformat
 

docs/changelog/136141.yaml

@@ -0,0 +1,6 @@
+pr: 136141
+summary: Add settings for health indicator `shard_capacity` thresholds
+area: Health
+type: enhancement
+issues:
+ - 116697

docs/changelog/136828.yaml

@@ -0,0 +1,5 @@
+pr: 136828
+summary: Can match phase coordinator duration APM metric
+area: Search
+type: enhancement
+issues: []

docs/changelog/136951.yaml

@@ -0,0 +1,5 @@
+pr: 136951
+summary: Support different downsampling methods through ILM
+area: "ILM+SLM"
+type: enhancement
+issues: []

docs/changelog/136996.yaml

@@ -0,0 +1,5 @@
+pr: 136996
+summary: Add periodic PKC JWK set reloading capability to JWT realm
+area: Security
+type: enhancement
+issues: []

docs/changelog/137222.yaml

@@ -0,0 +1,6 @@
+pr: 137222
+summary: "[Sentinel One] Add `manage`, `create_index`, `read`, `index`, `write`, `delete`, permission for third-party agent indices in the `Kibana system` to support the threat event data stream."
+area: Authorization
+type: enhancement
+issues:
+  - 240901