Skip to content

Add additional logging to make spotting stats issues easier #133972

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 6 commits into from
Sep 17, 2025
Merged

Add additional logging to make spotting stats issues easier #133972

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
483ba7f
Add additional logging to make spotting stats issues easier
lukewhiting Sep 2, 2025
f5864bd
Merge branch 'main' of github.com:elastic/elasticsearch into sdh-9234…
lukewhiting Sep 16, 2025
ba1bbaa
Refactor logging in JVM monitoring classes for improved error handling
lukewhiting Sep 16, 2025
8efbd2c
Merge branch 'main' into sdh-9234-win2019-stats-incorrect
lukewhiting Sep 16, 2025
eec20cd
Switch to debug level logging
lukewhiting Sep 17, 2025
715cc48
Merge branch 'main' into sdh-9234-win2019-stats-incorrect
lukewhiting Sep 17, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 8 additions & 0 deletions server/src/main/java/org/elasticsearch/monitor/Probes.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,21 +9,29 @@

package org.elasticsearch.monitor;

import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;

import java.lang.management.OperatingSystemMXBean;
import java.lang.reflect.Method;

public class Probes {
private static final Logger logger = LogManager.getLogger(Probes.class);

public static short getLoadAndScaleToPercent(Method method, OperatingSystemMXBean osMxBean) {
logger.debug("Starting probe of method {} on osMxBean {}", method, osMxBean);
if (method != null) {
try {
double load = (double) method.invoke(osMxBean);
if (load >= 0) {
return (short) (load * 100);
}
} catch (Exception e) {
logger.debug(() -> "failed to invoke method [" + method + "] on osMxBean [" + osMxBean + "]", e);
return -1;
}
}
logger.debug("Method is null. Returning default value.");
return -1;
}
}
41 changes: 31 additions & 10 deletions server/src/main/java/org/elasticsearch/monitor/jvm/JvmInfo.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@

package org.elasticsearch.monitor.jvm;

import org.apache.logging.log4j.Logger;
import org.elasticsearch.TransportVersions;
import org.elasticsearch.common.io.stream.StreamInput;
import org.elasticsearch.common.io.stream.StreamOutput;
Expand All @@ -30,10 +31,14 @@
import java.util.List;
import java.util.Map;

import static org.apache.logging.log4j.LogManager.getLogger;

public class JvmInfo implements ReportingService.Info {

private static final JvmInfo INSTANCE;

private static final Logger logger = getLogger(JvmInfo.class);

static {
RuntimeMXBean runtimeMXBean = ManagementFactory.getRuntimeMXBean();
MemoryMXBean memoryMXBean = ManagementFactory.getMemoryMXBean();
Expand Down Expand Up @@ -94,47 +99,63 @@ public class JvmInfo implements ReportingService.Info {
try {
Object onErrorObject = vmOptionMethod.invoke(hotSpotDiagnosticMXBean, "OnError");
onError = (String) valueMethod.invoke(onErrorObject);
} catch (Exception ignored) {}
} catch (Exception e) {
logger.debug("Error getting JVM info from MX Bean", e);
}

try {
Object onOutOfMemoryErrorObject = vmOptionMethod.invoke(hotSpotDiagnosticMXBean, "OnOutOfMemoryError");
onOutOfMemoryError = (String) valueMethod.invoke(onOutOfMemoryErrorObject);
} catch (Exception ignored) {}
} catch (Exception e) {
logger.debug("Error getting JVM info from MX Bean", e);
}

try {
Object useCompressedOopsVmOptionObject = vmOptionMethod.invoke(hotSpotDiagnosticMXBean, "UseCompressedOops");
useCompressedOops = (String) valueMethod.invoke(useCompressedOopsVmOptionObject);
} catch (Exception ignored) {}
} catch (Exception e) {
logger.debug("Error getting JVM info from MX Bean", e);
}

try {
Object useG1GCVmOptionObject = vmOptionMethod.invoke(hotSpotDiagnosticMXBean, "UseG1GC");
useG1GC = (String) valueMethod.invoke(useG1GCVmOptionObject);
Object regionSizeVmOptionObject = vmOptionMethod.invoke(hotSpotDiagnosticMXBean, "G1HeapRegionSize");
g1RegisionSize = Long.parseLong((String) valueMethod.invoke(regionSizeVmOptionObject));
} catch (Exception ignored) {}
} catch (Exception e) {
logger.debug("Error getting JVM info from MX Bean", e);
}

try {
Object initialHeapSizeVmOptionObject = vmOptionMethod.invoke(hotSpotDiagnosticMXBean, "InitialHeapSize");
configuredInitialHeapSize = Long.parseLong((String) valueMethod.invoke(initialHeapSizeVmOptionObject));
} catch (Exception ignored) {}
} catch (Exception e) {
logger.debug("Error getting JVM info from MX Bean", e);
}

try {
Object maxHeapSizeVmOptionObject = vmOptionMethod.invoke(hotSpotDiagnosticMXBean, "MaxHeapSize");
configuredMaxHeapSize = Long.parseLong((String) valueMethod.invoke(maxHeapSizeVmOptionObject));
} catch (Exception ignored) {}
} catch (Exception e) {
logger.debug("Error getting JVM info from MX Bean", e);
}

try {
Object maxDirectMemorySizeVmOptionObject = vmOptionMethod.invoke(hotSpotDiagnosticMXBean, "MaxDirectMemorySize");
directMemoryMax = Long.parseLong((String) valueMethod.invoke(maxDirectMemorySizeVmOptionObject));
} catch (Exception ignored) {}
} catch (Exception e) {
logger.debug("Error getting JVM info from MX Bean", e);
}

try {
Object useSerialGCVmOptionObject = vmOptionMethod.invoke(hotSpotDiagnosticMXBean, "UseSerialGC");
useSerialGC = (String) valueMethod.invoke(useSerialGCVmOptionObject);
} catch (Exception ignored) {}

} catch (Exception ignored) {
} catch (Exception e) {
logger.debug("Error getting JVM info from MX Bean", e);
}

} catch (Exception e) {
logger.debug("Error getting JVM info from MX Bean", e);
}

Mem mem = new Mem(heapInit, heapMax, nonHeapInit, nonHeapMax, directMemoryMax);
Expand Down
9 changes: 6 additions & 3 deletions server/src/main/java/org/elasticsearch/monitor/jvm/SunThreadInfo.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,21 +9,22 @@

package org.elasticsearch.monitor.jvm;

import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;

import java.lang.management.ManagementFactory;
import java.lang.management.ThreadMXBean;
import java.lang.reflect.Method;

import static org.apache.logging.log4j.LogManager.getLogger;

public class SunThreadInfo {

private static final ThreadMXBean threadMXBean;
private static final Method getThreadAllocatedBytes;
private static final Method isThreadAllocatedMemorySupported;
private static final Method isThreadAllocatedMemoryEnabled;

private static final Logger logger = LogManager.getLogger(SunThreadInfo.class);
private static final Logger logger = getLogger(SunThreadInfo.class);
public static final SunThreadInfo INSTANCE = new SunThreadInfo();

static {
Expand Down Expand Up @@ -83,11 +84,13 @@ public long getThreadAllocatedBytes(long id) {
}

private static Method getMethod(String methodName, Class<?>... parameterTypes) {
String className = "com.sun.management.ThreadMXBean";
try {
Method method = Class.forName("com.sun.management.ThreadMXBean").getMethod(methodName, parameterTypes);
Method method = Class.forName(className).getMethod(methodName, parameterTypes);
return method;
} catch (Exception e) {
// not available
logger.debug(() -> "failed to get method [" + methodName + "] from class [" + className + "]", e);
return null;
}
}
Expand Down
6 changes: 4 additions & 2 deletions server/src/main/java/org/elasticsearch/monitor/os/OsProbe.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -773,7 +773,7 @@ public static OsProbe getInstance() {

}

private final Logger logger = LogManager.getLogger(getClass());
private static final Logger logger = LogManager.getLogger(OsProbe.class);

OsInfo osInfo(long refreshInterval, Processors processors) throws IOException {
return new OsInfo(
Expand Down Expand Up @@ -913,10 +913,12 @@ public OsStats osStats() {
* Returns a given method of the OperatingSystemMXBean, or null if the method is not found or unavailable.
*/
private static Method getMethod(String methodName) {
String className = "com.sun.management.OperatingSystemMXBean";
try {
return Class.forName("com.sun.management.OperatingSystemMXBean").getMethod(methodName);
return Class.forName(className).getMethod(methodName);
} catch (Exception e) {
// not available
logger.debug(() -> "failed to get method [" + methodName + "] from class [" + className + "]", e);
return null;
}
}
Expand Down
16 changes: 12 additions & 4 deletions server/src/main/java/org/elasticsearch/monitor/process/ProcessProbe.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,17 +9,21 @@

package org.elasticsearch.monitor.process;

import org.apache.logging.log4j.Logger;
import org.elasticsearch.bootstrap.BootstrapInfo;
import org.elasticsearch.monitor.Probes;

import java.lang.management.ManagementFactory;
import java.lang.management.OperatingSystemMXBean;
import java.lang.reflect.Method;

import static org.apache.logging.log4j.LogManager.getLogger;
import static org.elasticsearch.monitor.jvm.JvmInfo.jvmInfo;

public class ProcessProbe {

private static final Logger logger = getLogger(ProcessProbe.class);

private static final OperatingSystemMXBean osMxBean = ManagementFactory.getOperatingSystemMXBean();

private static final Method getMaxFileDescriptorCountField;
Expand Down Expand Up @@ -130,10 +134,12 @@ public static ProcessStats processStats() {
* or null if the method is not found or unavailable.
*/
private static Method getMethod(String methodName) {
String className = "com.sun.management.OperatingSystemMXBean";
try {
return Class.forName("com.sun.management.OperatingSystemMXBean").getMethod(methodName);
} catch (Exception t) {
return Class.forName(className).getMethod(methodName);
} catch (Exception e) {
// not available
logger.debug(() -> "failed to get method [" + methodName + "] from class [" + className + "]", e);
return null;
}
}
Expand All @@ -143,10 +149,12 @@ private static Method getMethod(String methodName) {
* or null if the method is not found or unavailable.
*/
private static Method getUnixMethod(String methodName) {
String className = "com.sun.management.UnixOperatingSystemMXBean";
try {
return Class.forName("com.sun.management.UnixOperatingSystemMXBean").getMethod(methodName);
} catch (Exception t) {
return Class.forName(className).getMethod(methodName);
} catch (Exception e) {
// not available
logger.debug(() -> "failed to get method [" + methodName + "] from class [" + className + "]", e);
return null;
}
}
Expand Down