Set coredump filter for Elasticsearch (elastic#137300) (elastic#137445)

When a linux process dies and the core process memory is dumped, which
parts of the memory to output are controlled by the coredump filter. By
default the JVM dumps anonymous memory (ie heap and direct memory) and
file-backed memory (ie mmaps), and private huge pages. However, the
default of dumping file-backed memory for Elasticsearch is problematic;
these can be enormous, and they also aren't very useful since the
indices will likely be available when investigating the crash.

This commit sets the coredump filter when running on Linux to omit
file-backed memory from coredumps. Although the JVM has diagnostic VM
options for controlling the filter, they end up causing the filter to be
inherited, which is from the server CLI, another Java process, which has
the file-backed memory bits set. Instead of using the JVM options we
set it via the special proc filesystem.

Author: rjernst

qa/packaging/src/test/java/org/elasticsearch/packaging/test/LinuxSystemTests.java

@@ -0,0 +1,53 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+package org.elasticsearch.packaging.test;
+
+import org.elasticsearch.packaging.util.Platforms;
+import org.elasticsearch.packaging.util.ProcessInfo;
+import org.junit.BeforeClass;
+
+import java.util.List;
+
+import static com.carrotsearch.randomizedtesting.RandomizedTest.assumeTrue;
+import static org.hamcrest.Matchers.equalTo;
+
+// tests for how the linux distro interacts with the OS
+public class LinuxSystemTests extends PackagingTestCase {
+
+    @BeforeClass
+    public static void ensureLinux() {
+        assumeTrue(Platforms.LINUX);
+    }
+
+    public void test10Install() throws Exception {
+        install();
+    }
+
+    public void test20CoredumpFilter() throws Exception {
+        startElasticsearch();
+
+        // find the Elasticsearch process
+        int esPid = -1;
+        List<ProcessInfo> procs = ProcessInfo.getProcessInfo(sh, "java");
+        for (ProcessInfo proc : procs) {
+            if (proc.commandLine().contains("org.elasticsearch.bootstrap.Elasticsearch")) {
+                esPid = proc.pid();
+            }
+        }
+        if (esPid == -1) {
+            fail("Could not find Elasticsearch process, existing processes:\n" + procs);
+        }
+
+        // check the coredump filter was set correctly
+        String coredumpFilter = sh.run("cat /proc/" + esPid + "/coredump_filter").stdout().trim();
+        assertThat(coredumpFilter, equalTo("00000023"));
+    }
+
+}

qa/packaging/src/test/java/org/elasticsearch/packaging/util/ProcessInfo.java

@@ -22,7 +22,7 @@
  * works in Linux containers. At the moment that isn't a problem, because we only publish Docker images
  * for Linux.
  */
-public record ProcessInfo(int pid, int uid, int gid, String username, String group) {
+public record ProcessInfo(int pid, int uid, int gid, String username, String group, String commandLine) {
 
     /**
      * Fetches process information about <code>command</code>, using <code>sh</code> to execute commands.
@@ -53,7 +53,9 @@ public static List<ProcessInfo> getProcessInfo(Shell sh, String command) {
             final String username = sh.run("getent passwd " + uid + " | cut -f1 -d:").stdout().trim();
             final String group = sh.run("getent group " + gid + " | cut -f1 -d:").stdout().trim();
 
-            infos.add(new ProcessInfo(pid, uid, gid, username, group));
+            final String commandLine = sh.run("cat /proc/" + pid + "/cmdline").stdout().trim();
+
+            infos.add(new ProcessInfo(pid, uid, gid, username, group, commandLine));
         }
         return Collections.unmodifiableList(infos);
     }

server/src/main/java/org/elasticsearch/bootstrap/Elasticsearch.java

@@ -484,6 +484,10 @@ static void initializeNatives(final Path tmpFile, final boolean mlockAll, final
             }
         }
 
+        if (IOUtils.LINUX) {
+            setCoredumpFilter();
+        }
+
         // init lucene random seed. it will use /dev/urandom where available:
         StringHelper.randomId();
     }
@@ -598,6 +602,20 @@ static Map<String, Set<String>> findPluginsWithNativeAccess(Map<String, Policy>
         return pluginsWithNativeAccess;
     }
 
+    @SuppressForbidden(reason = "access proc filesystem")
+    private static void setCoredumpFilter() {
+        // The coredump filter determines which types of memory are added to core dumps. By default, Java
+        // includes memory mapped files, bits 2 and 3. Here we disable those bits. Note that the VM
+        // has special options to disable these, but the filter is then inherited from the parent process
+        // which is the server CLI, which is also a JVM so it has these bits set. Thus, we set it explicitly.
+        // See https://man7.org/linux/man-pages/man5/core.5.html for more info on the relevant bits of the filter
+        try {
+            Files.writeString(Path.of("/proc/self/coredump_filter"), "0x23");
+        } catch (IOException e) {
+            throw new RuntimeException("Could not set coredump filter", e);
+        }
+    }
+
     // -- instance
 
     private static volatile Elasticsearch INSTANCE;