logs for debugging heap size

Author: jan-elastic

distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/JvmOption.java

@@ -9,6 +9,8 @@
 
 package org.elasticsearch.server.cli;
 
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.core.SuppressForbidden;
 
@@ -33,6 +35,8 @@ class JvmOption {
     private final String value;
     private final String origin;
 
+    private static final Logger logger = LogManager.getLogger(JvmOption.class);
+
     JvmOption(String value, String origin) {
         if (origin == null) {
             throw new IllegalStateException(Strings.format("""
@@ -43,6 +47,11 @@ class JvmOption {
         this.origin = origin;
     }
 
+    @Override
+    public String toString() {
+        return "{value=" + value + ", origin=" + origin + "}";
+    }
+
     public Optional<String> getValue() {
         return Optional.ofNullable(value);
     }
@@ -124,6 +133,11 @@ private static List<String> flagsFinal(final List<String> userDefinedJvmOptions)
             );
             throw new RuntimeException(message);
         } else {
+            for (String line : output) {
+                if (line.contains("MaxHeapSize")) {
+                    logger.info("@@@ JvmOption flagsFinal: {}", line);
+                }
+            }
             return output;
         }
     }

distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/JvmOptionsParser.java

@@ -9,6 +9,8 @@
 
 package org.elasticsearch.server.cli;
 
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
 import org.elasticsearch.bootstrap.ServerArgs;
 import org.elasticsearch.cli.ExitCodes;
 import org.elasticsearch.cli.ProcessInfo;
@@ -43,6 +45,8 @@
  */
 public final class JvmOptionsParser {
 
+    private static final Logger logger = LogManager.getLogger(JvmOptionsParser.class);
+
     static class JvmOptionsFileParserException extends Exception {
 
         private final Path jvmOptionsFile;
@@ -176,6 +180,8 @@ List<String> readJvmOptionsFiles(final Path config) throws IOException, JvmOptio
         final List<String> jvmOptions = new ArrayList<>();
 
         for (final Path jvmOptionsFile : jvmOptionsFiles) {
+            logger.info("READING FILE: " + jvmOptionsFile);
+
             final SortedMap<Integer, String> invalidLines = new TreeMap<>();
             try (
                 InputStream is = Files.newInputStream(jvmOptionsFile);
@@ -187,6 +193,11 @@ List<String> readJvmOptionsFiles(final Path config) throws IOException, JvmOptio
             if (invalidLines.isEmpty() == false) {
                 throw new JvmOptionsFileParserException(jvmOptionsFile, invalidLines);
             }
+            for (String option : jvmOptions) {
+                if (option.contains("feature_flag")) {
+                    logger.info("FOUND OPTIONS: {}", option);
+                }
+            }
         }
         return jvmOptions;
     }

distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/MachineDependentHeap.java

@@ -9,6 +9,9 @@
 
 package org.elasticsearch.server.cli;
 
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.elasticsearch.cli.Terminal;
 import org.elasticsearch.cluster.node.DiscoveryNodeRole;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.FeatureFlag;
@@ -40,7 +43,19 @@ public class MachineDependentHeap {
 
     private static final FeatureFlag NEW_ML_MEMORY_COMPUTATION_FEATURE_FLAG = new FeatureFlag("new_ml_memory_computation");
 
-    public MachineDependentHeap() {}
+    private static final Logger logger = LogManager.getLogger(MachineDependentHeap.class);
+
+    private final Terminal terminal;
+
+    private boolean useNewMlMemoryComputation = false;
+
+    public MachineDependentHeap() {
+        this.terminal = null;
+    }
+
+    public MachineDependentHeap(Terminal terminal) {
+        this.terminal = terminal;
+    }
 
     /**
      * Calculate heap options.
@@ -55,8 +70,30 @@ public final List<String> determineHeapSettings(
         SystemMemoryInfo systemMemoryInfo,
         List<String> userDefinedJvmOptions
     ) throws IOException, InterruptedException {
+        if (userDefinedJvmOptions.contains("-Des.new_ml_memory_computation_feature_flag_enabled=true")
+                || NEW_ML_MEMORY_COMPUTATION_FEATURE_FLAG.isEnabled()) {
+            useNewMlMemoryComputation = true;
+        }
         // TODO: this could be more efficient, to only parse final options once
         final Map<String, JvmOption> finalJvmOptions = JvmOption.findFinalOptions(userDefinedJvmOptions);
+        if (terminal != null) {
+            terminal.println("@@@ contains -Des.new_ml_memory_computation_feature_flag_enabled=true --> "
+                + userDefinedJvmOptions.contains("-Des.new_ml_memory_computation_feature_flag_enabled=true"));
+            terminal.println("@@@ contains -Des.inference_default_elser_feature_flag_enabled=true --> "
+                + userDefinedJvmOptions.contains("-Des.inference_default_elser_feature_flag_enabled=true"));
+            terminal.println("@@@ MachineDependentHeap userDefinedJvmOptions: " + userDefinedJvmOptions);
+
+            terminal.println(
+                "@@@ MachineDependentHeap determineHeapSettings: MaxHeap="
+                    + finalJvmOptions.get("MaxHeapSize")
+                    + " FeatureFlag="
+                    + NEW_ML_MEMORY_COMPUTATION_FEATURE_FLAG.isEnabled()
+            );
+            terminal.println("@@@  MachineDependentHeap determineHeapSettings: inference_default_elser_feature_flag="
+                + new FeatureFlag("inference_default_elser").isEnabled());
+            terminal.println("@@@ USER DEFINED OPTIONS = " + userDefinedJvmOptions);
+            terminal.println("@@@ FINAL OPTIONS = " + finalJvmOptions);
+        }
         if (isMaxHeapSpecified(finalJvmOptions) || isMinHeapSpecified(finalJvmOptions) || isInitialHeapSpecified(finalJvmOptions)) {
             // User has explicitly set memory settings so we use those
             return Collections.emptyList();
@@ -65,6 +102,12 @@ public final List<String> determineHeapSettings(
         List<DiscoveryNodeRole> roles = NodeRoleSettings.NODE_ROLES_SETTING.get(nodeSettings);
         long availableSystemMemory = systemMemoryInfo.availableSystemMemory();
         MachineNodeRole nodeRole = mapNodeRole(roles);
+        if (terminal != null) {
+            terminal.println(
+                "@@@ MachineDependentHeap determineHeapSettings: setMaxHeap: "
+                    + getHeapSizeMb(nodeSettings, nodeRole, availableSystemMemory)
+            );
+        }
         return options(getHeapSizeMb(nodeSettings, nodeRole, availableSystemMemory));
     }
 
@@ -107,7 +150,7 @@ protected int getHeapSizeMb(Settings nodeSettings, MachineNodeRole role, long av
             case ML_ONLY -> {
                 double heapFractionBelow16GB = 0.4;
                 double heapFractionAbove16GB = 0.1;
-                if (NEW_ML_MEMORY_COMPUTATION_FEATURE_FLAG.isEnabled()) {
+                if (useNewMlMemoryComputation) {
                     heapFractionBelow16GB = 0.4 / (1.0 + JvmErgonomics.DIRECT_MEMORY_TO_HEAP_FACTOR);
                     heapFractionAbove16GB = 0.1 / (1.0 + JvmErgonomics.DIRECT_MEMORY_TO_HEAP_FACTOR);
                 }

distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/ServerCli.java

@@ -265,8 +265,13 @@ protected Command loadTool(Map<String, String> sysprops, String toolname, String
 
     // protected to allow tests to override
     protected ServerProcess startServer(Terminal terminal, ProcessInfo processInfo, ServerArgs args) throws Exception {
+        terminal.println("@@@ ServerCli startServer");
+
         var tempDir = ServerProcessUtils.setupTempDir(processInfo);
         var jvmOptions = JvmOptionsParser.determineJvmOptions(args, processInfo, tempDir, new MachineDependentHeap());
+
+        terminal.println("@@@ ServerCli startServer: jvmOptions=" + jvmOptions);
+
         var serverProcessBuilder = new ServerProcessBuilder().withTerminal(terminal)
             .withProcessInfo(processInfo)
             .withServerArgs(args)