Merge branch 'main' into exp-histo-topn

Author: JonasKunz

.buildkite/pipelines/periodic-packaging.template.yml

@@ -18,6 +18,7 @@ steps:
               - rocky-9
               - rhel-8
               - rhel-9
+              - rhel-10
               - almalinux-8
               - almalinux-9
         agents:

.buildkite/pipelines/periodic-packaging.yml

@@ -19,6 +19,7 @@ steps:
               - rocky-9
               - rhel-8
               - rhel-9
+              - rhel-10
               - almalinux-8
               - almalinux-9
         agents:

.buildkite/pipelines/periodic-platform-support.yml

@@ -18,6 +18,7 @@ steps:
               - rocky-9
               - rhel-8
               - rhel-9
+              - rhel-10
               - almalinux-8
               - almalinux-9
         agents:

.buildkite/pipelines/pull-request/packaging-tests-unix.yml

@@ -21,6 +21,7 @@ steps:
               - rocky-9
               - rhel-8
               - rhel-9
+              - rhel-10
               - almalinux-8
               - almalinux-9
             PACKAGING_TASK:

build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/transport/TransportVersionGenerationFuncTest.groovy

@@ -513,4 +513,19 @@ class TransportVersionGenerationFuncTest extends AbstractTransportVersionFuncTes
         assertGenerateAndValidateSuccess(result)
         assertUpperBound("9.2", "existing_92,8123000")
     }
+
+    def "generation cannot run on release branch"() {
+        given:
+        file("myserver/build.gradle") << """
+            tasks.named('generateTransportVersion') {
+                currentUpperBoundName = '9.1'
+            }
+        """
+
+        when:
+        def result = runGenerateTask().buildAndFail()
+
+        then:
+        assertGenerateFailure(result, "Transport version generation cannot run on release branches")
+    }
 }

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/ElasticsearchBuildCompletePlugin.java

@@ -48,6 +48,7 @@
 import java.util.Arrays;
 import java.util.List;
 import java.util.Optional;
+import java.util.concurrent.TimeUnit;
 
 import javax.inject.Inject;
 
@@ -199,7 +200,11 @@ public void execute(BuildFinishedFlowAction.Parameters parameters) throws FileNo
                     try {
                         // we are very generious here, as the upload can take
                         // a long time depending on its size
-                        pb.start().waitFor(30, java.util.concurrent.TimeUnit.MINUTES);
+                        long timeoutSec = calculateUploadWaitTimeoutSeconds(uploadFile);
+                        boolean completedInTime = pb.start().waitFor(timeoutSec, TimeUnit.SECONDS);
+                        if (completedInTime == false) {
+                            System.out.println("Timed out waiting for buildkite artifact upload after " + timeoutSec + " seconds");
+                        }
                     } catch (InterruptedException e) {
                         System.out.println("Failed to upload buildkite artifact " + e.getMessage());
                     }
@@ -304,5 +309,14 @@ private static String calculateArchivePath(Path path, Path projectPath) {
             }
             return archivePath;
         }
+
+        private static long calculateUploadWaitTimeoutSeconds(File file) {
+            long fileSizeBytes = file.length();
+            long fileSizeMB = fileSizeBytes / (1024 * 1024);
+
+            // Allocate 4 seconds per MB (assumes ~250 KB/s upload speed)
+            // with min 10 seconds and max 30 minutes
+            return Math.max(10, Math.min(1800, fileSizeMB * 4));
+        }
     }
 }

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GenerateTransportVersionDefinitionTask.java

@@ -96,6 +96,12 @@ public abstract class GenerateTransportVersionDefinitionTask extends DefaultTask
     @TaskAction
     public void run() throws IOException {
         TransportVersionResourcesService resources = getResourceService().get();
+        List<TransportVersionUpperBound> upstreamUpperBounds = resources.getUpperBoundsFromGitBase();
+        boolean onReleaseBranch = resources.checkIfDefinitelyOnReleaseBranch(upstreamUpperBounds, getCurrentUpperBoundName().get());
+        if (onReleaseBranch) {
+            throw new IllegalArgumentException("Transport version generation cannot run on release branches");
+        }
+
         Set<String> referencedNames = TransportVersionReference.collectNames(getReferencesFiles());
         Set<String> changedDefinitionNames = resources.getChangedReferableDefinitionNames();
         String targetDefinitionName = getTargetDefinitionName(resources, referencedNames, changedDefinitionNames);
@@ -109,7 +115,7 @@ public void run() throws IOException {
             resetAllUpperBounds(resources, idsByBase);
         } else {
             getLogger().lifecycle("Generating transport version name: " + targetDefinitionName);
-            List<TransportVersionUpperBound> upstreamUpperBounds = resources.getUpperBoundsFromGitBase();
+
             Set<String> targetUpperBoundNames = getTargetUpperBoundNames(resources, upstreamUpperBounds, targetDefinitionName);
 
             List<TransportVersionId> ids = updateUpperBounds(

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionResourcesService.java

@@ -25,6 +25,7 @@
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.util.ArrayList;
+import java.util.Collection;
 import java.util.Collections;
 import java.util.Comparator;
 import java.util.HashMap;
@@ -259,6 +260,20 @@ private Path getUpperBoundRelativePath(String name) {
         return UPPER_BOUNDS_DIR.resolve(name + ".csv");
     }
 
+    boolean checkIfDefinitelyOnReleaseBranch(Collection<TransportVersionUpperBound> upperBounds, String currentUpperBoundName) {
+        // only want to look at definitions <= the current upper bound.
+        // TODO: we should filter all of the upper bounds/definitions that are validated by this, not just in this method
+        TransportVersionUpperBound currentUpperBound = upperBounds.stream()
+            .filter(u -> u.name().equals(currentUpperBoundName))
+            .findFirst()
+            .orElse(null);
+        if (currentUpperBound == null) {
+            // since there is no current upper bound, we don't know if we are on a release branch
+            return false;
+        }
+        return upperBounds.stream().anyMatch(u -> u.definitionId().complete() > currentUpperBound.definitionId().complete());
+    }
+
     private String getBaseRefName() {
         if (baseRefName.get() == null) {
             synchronized (baseRefName) {

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/ValidateTransportVersionResourcesTask.java

@@ -84,7 +84,7 @@ public void validateTransportVersions() throws IOException {
         Map<Integer, List<IdAndDefinition>> idsByBase = resources.getIdsByBase();
         Map<String, TransportVersionUpperBound> upperBounds = resources.getUpperBounds();
         TransportVersionUpperBound currentUpperBound = upperBounds.get(getCurrentUpperBoundName().get());
-        boolean onReleaseBranch = checkIfDefinitelyOnReleaseBranch(upperBounds);
+        boolean onReleaseBranch = resources.checkIfDefinitelyOnReleaseBranch(upperBounds.values(), getCurrentUpperBoundName().get());
         boolean validateModifications = onReleaseBranch == false || getCI().get();
 
         for (var definition : referableDefinitions.values()) {
@@ -330,15 +330,6 @@ private void validatePrimaryIds(
         );
     }
 
-    private boolean checkIfDefinitelyOnReleaseBranch(Map<String, TransportVersionUpperBound> upperBounds) {
-        // only want to look at definitions <= the current upper bound.
-        // TODO: we should filter all of the upper bounds/definitions that are validated by this, not just in this method
-        String currentUpperBoundName = getCurrentUpperBoundName().get();
-        TransportVersionUpperBound currentUpperBound = upperBounds.get(currentUpperBoundName);
-
-        return upperBounds.values().stream().anyMatch(u -> u.definitionId().complete() > currentUpperBound.definitionId().complete());
-    }
-
     private void throwDefinitionFailure(TransportVersionDefinition definition, String message) {
         Path relativePath = getResources().get().getDefinitionPath(definition);
         throw new VerificationException("Transport version definition file [" + relativePath + "] " + message);

build-tools/src/main/java/org/elasticsearch/gradle/testclusters/MockApmServer.java

@@ -9,8 +9,11 @@
 
 package org.elasticsearch.gradle.testclusters;
 
+import com.fasterxml.jackson.databind.JsonNode;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.fasterxml.jackson.databind.node.ObjectNode;
+import com.fasterxml.jackson.databind.util.LRUMap;
+import com.fasterxml.jackson.databind.util.LookupCache;
 import com.sun.net.httpserver.HttpExchange;
 import com.sun.net.httpserver.HttpHandler;
 import com.sun.net.httpserver.HttpServer;
@@ -28,7 +31,9 @@
 import java.io.InputStreamReader;
 import java.io.OutputStream;
 import java.net.InetSocketAddress;
+import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.List;
 import java.util.regex.Pattern;
 import java.util.stream.Collectors;
 
@@ -48,6 +53,8 @@
 public class MockApmServer {
     private static final Logger logger = Logging.getLogger(MockApmServer.class);
     private static final org.slf4j.Logger log = LoggerFactory.getLogger(MockApmServer.class);
+    private static final LookupCache<String, String> transactionCache = new LRUMap(16, 16);
+
     private final Pattern metricFilter;
     private final Pattern transactionFilter;
     private final Pattern transactionExcludesFilter;
@@ -136,22 +143,28 @@ private void logFiltered(InputStream body) throws IOException {
             ObjectMapper mapper = new ObjectMapper();
             try (BufferedReader reader = new BufferedReader(new InputStreamReader(body))) {
                 String line;
-                String tier = null;
-                String node = null;
+                String nodeMetadata = null;
+
+                List<JsonNode> spans = new ArrayList<>();
 
                 while ((line = reader.readLine()) != null) {
                     var jsonNode = mapper.readTree(line);
 
                     if (jsonNode.has("metadata")) {
-                        node = jsonNode.path("metadata").path("service").path("node").path("configured_name").asText(null);
-                        tier = jsonNode.path("metadata").path("labels").path("node_tier").asText(null);
+                        nodeMetadata = jsonNode.path("metadata").path("service").path("node").path("configured_name").asText(null);
+                        var tier = jsonNode.path("metadata").path("labels").path("node_tier").asText(null);
+                        nodeMetadata += tier != null ? "/" + tier : "";
+
                     } else if (transactionFilter != null && jsonNode.has("transaction")) {
                         var transaction = jsonNode.get("transaction");
                         var name = transaction.get("name").asText();
                         if (transactionFilter.matcher(name).matches()
                             && (transactionExcludesFilter == null || transactionExcludesFilter.matcher(name).matches() == false)) {
-                            logger.lifecycle("Transaction [{}/{}]: {}", node, tier, transaction);
+                            transactionCache.put(transaction.get("id").asText(), name);
+                            logger.lifecycle("Transaction {} [{}]: {}", name, nodeMetadata, transaction);
                         }
+                    } else if (jsonNode.has("span")) {
+                        spans.add(jsonNode.get("span")); // make sure to record all transactions first
                     } else if (metricFilter != null && jsonNode.has("metricset")) {
                         var metricset = jsonNode.get("metricset");
                         var samples = (ObjectNode) metricset.get("samples");
@@ -161,10 +174,20 @@ private void logFiltered(InputStream body) throws IOException {
                             }
                         }
                         if (samples.isEmpty() == false) {
-                            logger.lifecycle("Metricset [{}/{}]", node, tier, metricset);
+                            logger.lifecycle("Metricset [{}]: {}", nodeMetadata, metricset);
                         }
                     }
                 }
+
+                // emit only spans for previously matched transactions using the transaction cache
+                for (var span : spans) {
+                    var name = span.get("name").asText();
+                    var transactionId = span.get("transaction_id").asText();
+                    var transactionName = transactionCache.get(transactionId);
+                    if (transactionName != null) {
+                        logger.lifecycle("Span {} of {} [{}]: {}", name, transactionName, nodeMetadata, span);
+                    }
+                }
             }
         }
     }