Merge branch '9.1' into backport/9.1/pr-137222

Author: mohitjha-elastic

build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/transport/TransportVersionGenerationFuncTest.groovy

@@ -513,4 +513,19 @@ class TransportVersionGenerationFuncTest extends AbstractTransportVersionFuncTes
         assertGenerateAndValidateSuccess(result)
         assertUpperBound("9.2", "existing_92,8123000")
     }
+
+    def "generation cannot run on release branch"() {
+        given:
+        file("myserver/build.gradle") << """
+            tasks.named('generateTransportVersion') {
+                currentUpperBoundName = '9.1'
+            }
+        """
+
+        when:
+        def result = runGenerateTask().buildAndFail()
+
+        then:
+        assertGenerateFailure(result, "Transport version generation cannot run on release branches")
+    }
 }

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/ElasticsearchBuildCompletePlugin.java

@@ -44,6 +44,7 @@
 import java.util.Arrays;
 import java.util.List;
 import java.util.Optional;
+import java.util.concurrent.TimeUnit;
 
 import javax.inject.Inject;
 
@@ -178,7 +179,11 @@ public void execute(BuildFinishedFlowAction.Parameters parameters) throws FileNo
                     try {
                         // we are very generious here, as the upload can take
                         // a long time depending on its size
-                        pb.start().waitFor(30, java.util.concurrent.TimeUnit.MINUTES);
+                        long timeoutSec = calculateUploadWaitTimeoutSeconds(uploadFile);
+                        boolean completedInTime = pb.start().waitFor(timeoutSec, TimeUnit.SECONDS);
+                        if (completedInTime == false) {
+                            System.out.println("Timed out waiting for buildkite artifact upload after " + timeoutSec + " seconds");
+                        }
                     } catch (InterruptedException e) {
                         System.out.println("Failed to upload buildkite artifact " + e.getMessage());
                     }
@@ -278,5 +283,14 @@ private static void createBuildArchiveTar(List<File> files, File projectDir, Fil
         private static String calculateArchivePath(Path path, Path projectPath) {
             return path.startsWith(projectPath) ? projectPath.relativize(path).toString() : path.getFileName().toString();
         }
+
+        private static long calculateUploadWaitTimeoutSeconds(File file) {
+            long fileSizeBytes = file.length();
+            long fileSizeMB = fileSizeBytes / (1024 * 1024);
+
+            // Allocate 4 seconds per MB (assumes ~250 KB/s upload speed)
+            // with min 10 seconds and max 30 minutes
+            return Math.max(10, Math.min(1800, fileSizeMB * 4));
+        }
     }
 }

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GenerateTransportVersionDefinitionTask.java

@@ -96,6 +96,12 @@ public abstract class GenerateTransportVersionDefinitionTask extends DefaultTask
     @TaskAction
     public void run() throws IOException {
         TransportVersionResourcesService resources = getResourceService().get();
+        List<TransportVersionUpperBound> upstreamUpperBounds = resources.getUpperBoundsFromGitBase();
+        boolean onReleaseBranch = resources.checkIfDefinitelyOnReleaseBranch(upstreamUpperBounds, getCurrentUpperBoundName().get());
+        if (onReleaseBranch) {
+            throw new IllegalArgumentException("Transport version generation cannot run on release branches");
+        }
+
         Set<String> referencedNames = TransportVersionReference.collectNames(getReferencesFiles());
         Set<String> changedDefinitionNames = resources.getChangedReferableDefinitionNames();
         String targetDefinitionName = getTargetDefinitionName(resources, referencedNames, changedDefinitionNames);
@@ -109,7 +115,7 @@ public void run() throws IOException {
             resetAllUpperBounds(resources, idsByBase);
         } else {
             getLogger().lifecycle("Generating transport version name: " + targetDefinitionName);
-            List<TransportVersionUpperBound> upstreamUpperBounds = resources.getUpperBoundsFromGitBase();
+
             Set<String> targetUpperBoundNames = getTargetUpperBoundNames(resources, upstreamUpperBounds, targetDefinitionName);
 
             List<TransportVersionId> ids = updateUpperBounds(

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionResourcesService.java

@@ -25,6 +25,7 @@
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.util.ArrayList;
+import java.util.Collection;
 import java.util.Collections;
 import java.util.Comparator;
 import java.util.HashMap;
@@ -259,6 +260,20 @@ private Path getUpperBoundRelativePath(String name) {
         return UPPER_BOUNDS_DIR.resolve(name + ".csv");
     }
 
+    boolean checkIfDefinitelyOnReleaseBranch(Collection<TransportVersionUpperBound> upperBounds, String currentUpperBoundName) {
+        // only want to look at definitions <= the current upper bound.
+        // TODO: we should filter all of the upper bounds/definitions that are validated by this, not just in this method
+        TransportVersionUpperBound currentUpperBound = upperBounds.stream()
+            .filter(u -> u.name().equals(currentUpperBoundName))
+            .findFirst()
+            .orElse(null);
+        if (currentUpperBound == null) {
+            // since there is no current upper bound, we don't know if we are on a release branch
+            return false;
+        }
+        return upperBounds.stream().anyMatch(u -> u.definitionId().complete() > currentUpperBound.definitionId().complete());
+    }
+
     private String getBaseRefName() {
         if (baseRefName.get() == null) {
             synchronized (baseRefName) {

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/ValidateTransportVersionResourcesTask.java

@@ -84,7 +84,7 @@ public void validateTransportVersions() throws IOException {
         Map<Integer, List<IdAndDefinition>> idsByBase = resources.getIdsByBase();
         Map<String, TransportVersionUpperBound> upperBounds = resources.getUpperBounds();
         TransportVersionUpperBound currentUpperBound = upperBounds.get(getCurrentUpperBoundName().get());
-        boolean onReleaseBranch = checkIfDefinitelyOnReleaseBranch(upperBounds);
+        boolean onReleaseBranch = resources.checkIfDefinitelyOnReleaseBranch(upperBounds.values(), getCurrentUpperBoundName().get());
         boolean validateModifications = onReleaseBranch == false || getCI().get();
 
         for (var definition : referableDefinitions.values()) {
@@ -330,15 +330,6 @@ private void validatePrimaryIds(
         );
     }
 
-    private boolean checkIfDefinitelyOnReleaseBranch(Map<String, TransportVersionUpperBound> upperBounds) {
-        // only want to look at definitions <= the current upper bound.
-        // TODO: we should filter all of the upper bounds/definitions that are validated by this, not just in this method
-        String currentUpperBoundName = getCurrentUpperBoundName().get();
-        TransportVersionUpperBound currentUpperBound = upperBounds.get(currentUpperBoundName);
-
-        return upperBounds.values().stream().anyMatch(u -> u.definitionId().complete() > currentUpperBound.definitionId().complete());
-    }
-
     private void throwDefinitionFailure(TransportVersionDefinition definition, String message) {
         Path relativePath = getResources().get().getDefinitionPath(definition);
         throw new VerificationException("Transport version definition file [" + relativePath + "] " + message);

docs/changelog/137442.yaml

@@ -0,0 +1,5 @@
+pr: 137442
+summary: Handle ._original stored fields with fls
+area: "Authorization"
+type: bug
+issues: []

modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/MatchOnlyTextFieldMapper.java

@@ -208,7 +208,7 @@ public MatchOnlyTextFieldType(
             super(name, true, false, false, tsi, meta);
             this.indexAnalyzer = Objects.requireNonNull(indexAnalyzer);
             this.textFieldType = new TextFieldType(name, isSyntheticSource, syntheticSourceDelegate);
-            this.originalName = isSyntheticSource ? name + "._original" : null;
+            this.originalName = isSyntheticSource ? name + KeywordFieldMapper.FALLBACK_FIELD_NAME_SUFFIX : null;
             this.withinMultiField = withinMultiField;
             this.storedFieldInBinaryFormat = storedFieldInBinaryFormat;
         }

qa/packaging/src/test/java/org/elasticsearch/packaging/test/LinuxSystemTests.java

@@ -0,0 +1,53 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+package org.elasticsearch.packaging.test;
+
+import org.elasticsearch.packaging.util.Platforms;
+import org.elasticsearch.packaging.util.ProcessInfo;
+import org.junit.BeforeClass;
+
+import java.util.List;
+
+import static com.carrotsearch.randomizedtesting.RandomizedTest.assumeTrue;
+import static org.hamcrest.Matchers.equalTo;
+
+// tests for how the linux distro interacts with the OS
+public class LinuxSystemTests extends PackagingTestCase {
+
+    @BeforeClass
+    public static void ensureLinux() {
+        assumeTrue(Platforms.LINUX);
+    }
+
+    public void test10Install() throws Exception {
+        install();
+    }
+
+    public void test20CoredumpFilter() throws Exception {
+        startElasticsearch();
+
+        // find the Elasticsearch process
+        int esPid = -1;
+        List<ProcessInfo> procs = ProcessInfo.getProcessInfo(sh, "java");
+        for (ProcessInfo proc : procs) {
+            if (proc.commandLine().contains("org.elasticsearch.bootstrap.Elasticsearch")) {
+                esPid = proc.pid();
+            }
+        }
+        if (esPid == -1) {
+            fail("Could not find Elasticsearch process, existing processes:\n" + procs);
+        }
+
+        // check the coredump filter was set correctly
+        String coredumpFilter = sh.run("cat /proc/" + esPid + "/coredump_filter").stdout().trim();
+        assertThat(coredumpFilter, equalTo("00000023"));
+    }
+
+}

qa/packaging/src/test/java/org/elasticsearch/packaging/util/ProcessInfo.java

@@ -22,7 +22,7 @@
  * works in Linux containers. At the moment that isn't a problem, because we only publish Docker images
  * for Linux.
  */
-public record ProcessInfo(int pid, int uid, int gid, String username, String group) {
+public record ProcessInfo(int pid, int uid, int gid, String username, String group, String commandLine) {
 
     /**
      * Fetches process information about <code>command</code>, using <code>sh</code> to execute commands.
@@ -53,7 +53,9 @@ public static List<ProcessInfo> getProcessInfo(Shell sh, String command) {
             final String username = sh.run("getent passwd " + uid + " | cut -f1 -d:").stdout().trim();
             final String group = sh.run("getent group " + gid + " | cut -f1 -d:").stdout().trim();
 
-            infos.add(new ProcessInfo(pid, uid, gid, username, group));
+            final String commandLine = sh.run("cat /proc/" + pid + "/cmdline").stdout().trim();
+
+            infos.add(new ProcessInfo(pid, uid, gid, username, group, commandLine));
         }
         return Collections.unmodifiableList(infos);
     }

server/src/main/java/org/elasticsearch/bootstrap/Elasticsearch.java

@@ -484,6 +484,10 @@ static void initializeNatives(final Path tmpFile, final boolean mlockAll, final
             }
         }
 
+        if (IOUtils.LINUX) {
+            setCoredumpFilter();
+        }
+
         // init lucene random seed. it will use /dev/urandom where available:
         StringHelper.randomId();
     }
@@ -598,6 +602,20 @@ static Map<String, Set<String>> findPluginsWithNativeAccess(Map<String, Policy>
         return pluginsWithNativeAccess;
     }
 
+    @SuppressForbidden(reason = "access proc filesystem")
+    private static void setCoredumpFilter() {
+        // The coredump filter determines which types of memory are added to core dumps. By default, Java
+        // includes memory mapped files, bits 2 and 3. Here we disable those bits. Note that the VM
+        // has special options to disable these, but the filter is then inherited from the parent process
+        // which is the server CLI, which is also a JVM so it has these bits set. Thus, we set it explicitly.
+        // See https://man7.org/linux/man-pages/man5/core.5.html for more info on the relevant bits of the filter
+        try {
+            Files.writeString(Path.of("/proc/self/coredump_filter"), "0x23");
+        } catch (IOException e) {
+            throw new RuntimeException("Could not set coredump filter", e);
+        }
+    }
+
     // -- instance
 
     private static volatile Elasticsearch INSTANCE;