From 265ee485b235e084f75131b87ec55202e0e4a5c7 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 25 Oct 2025 08:05:18 +0000 Subject: [PATCH 1/4] Fix memory leak by limiting internal message queue size MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This commit addresses GitHub issue #61 where memory usage keeps increasing and garbage collection fails, eventually causing node crashes after several days. Root Cause: The syslog4j library queues messages internally with no limit by default when using TCP/SSL protocols. Under high message throughput, this causes unbounded memory growth leading to OOM errors. Solution: 1. Added maxQueueSize configuration parameter (default: 500 messages) 2. Set maxQueueSize on all syslog config objects (UDP/TCP/SSL) 3. Made the parameter user-configurable through Graylog UI The conservative default of 500 prevents memory leaks while still allowing reasonable buffering. Users experiencing high throughput can tune this value based on their needs, but should avoid unlimited queuing (-1). When the queue is full, the plugin will block until space is available, providing natural backpressure instead of consuming all available memory. Configuration: - Default: 500 messages - Configurable via "Maximum queue size" field in output configuration - Set to -1 for unlimited (not recommended, will cause OOM) Fixes #61 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude --- .../graylog2/plugin/SyslogOutput.java | 20 +++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/src/main/java/com/wizecore/graylog2/plugin/SyslogOutput.java b/src/main/java/com/wizecore/graylog2/plugin/SyslogOutput.java index d8750c5..f8cbfb9 100644 --- a/src/main/java/com/wizecore/graylog2/plugin/SyslogOutput.java +++ b/src/main/java/com/wizecore/graylog2/plugin/SyslogOutput.java @@ -18,6 +18,7 @@ import org.graylog2.syslog4j.SyslogConfigIF; import org.graylog2.syslog4j.SyslogIF; import org.graylog2.syslog4j.SyslogRuntimeException; +import org.graylog2.syslog4j.impl.AbstractSyslogConfigIF; import org.graylog2.syslog4j.impl.message.processor.SyslogMessageProcessor; import org.graylog2.syslog4j.impl.message.processor.structured.StructuredSyslogMessageProcessor; import org.graylog2.syslog4j.impl.net.tcp.TCPNetSyslogConfig; @@ -143,6 +144,23 @@ public SyslogOutput(@Assisted Stream stream, @Assisted Configuration conf) { config.setMaxMessageLength(maxlen); config.setTruncateMessage(true); + // Set maximum queue size to prevent OOM under high load (issue #61) + // Default is unlimited which causes memory leaks + int maxQueueSize = 500; // Conservative default + try { + String queueSizeStr = conf.getString("maxQueueSize"); + if (queueSizeStr != null && !queueSizeStr.trim().isEmpty()) { + maxQueueSize = Integer.parseInt(queueSizeStr); + } + } catch (Exception e) { + log.warning("Failed to parse maxQueueSize, using default: " + maxQueueSize); + } + // setMaxQueueSize is available on AbstractSyslogConfigIF, which all configs implement + if (config instanceof AbstractSyslogConfigIF) { + ((AbstractSyslogConfigIF) config).setMaxQueueSize(maxQueueSize); + log.info("Set maxQueueSize to " + maxQueueSize + " for " + protocol + "://" + host + ":" + port); + } + String hash = protocol + "_" + host + "_" + port + "_" + format; syslog = Syslog.exists(hash) ? Syslog.getInstance(hash) : Syslog.createInstance(hash, config); @@ -320,6 +338,8 @@ public ConfigurationRequest getRequestedConfiguration() { configurationRequest.addField(new TextField("maxlen", "Maximum message length", "", "Maximum message (body) length. Longer messages will be truncated. If not specified defaults to 16384 bytes.", ConfigurationField.Optional.OPTIONAL)); + configurationRequest.addField(new TextField("maxQueueSize", "Maximum queue size", "500", "Maximum number of messages to queue internally before blocking (prevents OOM). Set to -1 for unlimited (not recommended). Default is 500.", ConfigurationField.Optional.OPTIONAL)); + configurationRequest.addField(new TextField("keystore", "Key store", "", "Path to Java keystore (required for SSL over TCP). Must contain private key and cert for this client.", ConfigurationField.Optional.OPTIONAL)); configurationRequest.addField(new TextField("keystorePassword", "Key store password", "", "", ConfigurationField.Optional.OPTIONAL)); From cca39a047b7d03c288501538748f19122da7f166 Mon Sep 17 00:00:00 2001 From: Ruslan Gainutdinov Date: Sat, 25 Oct 2025 13:36:32 +0300 Subject: [PATCH 2/4] Clarify maxQueueSize field description Updated the description for maxQueueSize field to clarify its purpose. --- src/main/java/com/wizecore/graylog2/plugin/SyslogOutput.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/com/wizecore/graylog2/plugin/SyslogOutput.java b/src/main/java/com/wizecore/graylog2/plugin/SyslogOutput.java index f8cbfb9..ada178e 100644 --- a/src/main/java/com/wizecore/graylog2/plugin/SyslogOutput.java +++ b/src/main/java/com/wizecore/graylog2/plugin/SyslogOutput.java @@ -338,7 +338,7 @@ public ConfigurationRequest getRequestedConfiguration() { configurationRequest.addField(new TextField("maxlen", "Maximum message length", "", "Maximum message (body) length. Longer messages will be truncated. If not specified defaults to 16384 bytes.", ConfigurationField.Optional.OPTIONAL)); - configurationRequest.addField(new TextField("maxQueueSize", "Maximum queue size", "500", "Maximum number of messages to queue internally before blocking (prevents OOM). Set to -1 for unlimited (not recommended). Default is 500.", ConfigurationField.Optional.OPTIONAL)); + configurationRequest.addField(new TextField("maxQueueSize", "Maximum queue size", "500", "Maximum number of messages to queue internally. Set to -1 for unlimited. Default is 500.", ConfigurationField.Optional.OPTIONAL)); configurationRequest.addField(new TextField("keystore", "Key store", "", "Path to Java keystore (required for SSL over TCP). Must contain private key and cert for this client.", ConfigurationField.Optional.OPTIONAL)); configurationRequest.addField(new TextField("keystorePassword", "Key store password", "", "", ConfigurationField.Optional.OPTIONAL)); From 9beb482d2b1094730d3a56347a11c402f3530550 Mon Sep 17 00:00:00 2001 From: Ruslan Gainutdinov Date: Sat, 25 Oct 2025 13:40:35 +0300 Subject: [PATCH 3/4] Add maxQueueSize configuration for syslog output --- .../graylog2/plugin/SyslogOutput.java | 32 ++++++++----------- 1 file changed, 13 insertions(+), 19 deletions(-) diff --git a/src/main/java/com/wizecore/graylog2/plugin/SyslogOutput.java b/src/main/java/com/wizecore/graylog2/plugin/SyslogOutput.java index ada178e..9ba17a8 100644 --- a/src/main/java/com/wizecore/graylog2/plugin/SyslogOutput.java +++ b/src/main/java/com/wizecore/graylog2/plugin/SyslogOutput.java @@ -93,13 +93,23 @@ public SyslogOutput(@Assisted Stream stream, @Assisted Configuration conf) { format = "plain"; } + int maxQueueSize = 500; + String queueSizeStr = conf.getString("maxQueueSize"); + if (queueSizeStr != null && !queueSizeStr.trim().isEmpty()) { + maxQueueSize = Integer.parseInt(queueSizeStr); + } + log.info("Creating syslog output " + protocol + "://" + host + ":" + port + ", format " + format); SyslogConfigIF config = null; if (protocol.toLowerCase().equals("udp")) { - config = new UDPNetSyslogConfig(); + UDPNetSyslogConfig updConfig = new UDPNetSyslogConfig(); + udpConfig.setMaxQueueSize(maxQueueSize); + config = udpConfig; } else if (protocol.toLowerCase().equals("tcp")) { - config = new TCPNetSyslogConfig(); + TCPNetSyslogConfig tcpConfig = new TCPNetSyslogConfig(); + tcpConfig.setMaxQueueSize(maxQueueSize); + config = tcpConfig; } else if (protocol.toLowerCase().equals("tcp-ssl")) { CustomSSLSyslogConfig sslConfig = new CustomSSLSyslogConfig(); @@ -129,6 +139,7 @@ public SyslogOutput(@Assisted Stream stream, @Assisted Configuration conf) { sslConfig.setKeyStorePassword(ksp); sslConfig.setTrustStore(ts); sslConfig.setTrustStorePassword(tsp); + sslConfig.setMaxQueueSize(maxQueueSize); } else { throw new IllegalArgumentException("Unknown protocol: " + protocol); } @@ -144,23 +155,6 @@ public SyslogOutput(@Assisted Stream stream, @Assisted Configuration conf) { config.setMaxMessageLength(maxlen); config.setTruncateMessage(true); - // Set maximum queue size to prevent OOM under high load (issue #61) - // Default is unlimited which causes memory leaks - int maxQueueSize = 500; // Conservative default - try { - String queueSizeStr = conf.getString("maxQueueSize"); - if (queueSizeStr != null && !queueSizeStr.trim().isEmpty()) { - maxQueueSize = Integer.parseInt(queueSizeStr); - } - } catch (Exception e) { - log.warning("Failed to parse maxQueueSize, using default: " + maxQueueSize); - } - // setMaxQueueSize is available on AbstractSyslogConfigIF, which all configs implement - if (config instanceof AbstractSyslogConfigIF) { - ((AbstractSyslogConfigIF) config).setMaxQueueSize(maxQueueSize); - log.info("Set maxQueueSize to " + maxQueueSize + " for " + protocol + "://" + host + ":" + port); - } - String hash = protocol + "_" + host + "_" + port + "_" + format; syslog = Syslog.exists(hash) ? Syslog.getInstance(hash) : Syslog.createInstance(hash, config); From 4e33becf8ff220e7ea9fde3ff9e219abdbd7f8c3 Mon Sep 17 00:00:00 2001 From: Ruslan Gainutdinov Date: Sat, 25 Oct 2025 13:42:24 +0300 Subject: [PATCH 4/4] Fix typo in variable name for UDP config --- src/main/java/com/wizecore/graylog2/plugin/SyslogOutput.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/com/wizecore/graylog2/plugin/SyslogOutput.java b/src/main/java/com/wizecore/graylog2/plugin/SyslogOutput.java index 9ba17a8..bd8b006 100644 --- a/src/main/java/com/wizecore/graylog2/plugin/SyslogOutput.java +++ b/src/main/java/com/wizecore/graylog2/plugin/SyslogOutput.java @@ -102,7 +102,7 @@ public SyslogOutput(@Assisted Stream stream, @Assisted Configuration conf) { log.info("Creating syslog output " + protocol + "://" + host + ":" + port + ", format " + format); SyslogConfigIF config = null; if (protocol.toLowerCase().equals("udp")) { - UDPNetSyslogConfig updConfig = new UDPNetSyslogConfig(); + UDPNetSyslogConfig udpConfig = new UDPNetSyslogConfig(); udpConfig.setMaxQueueSize(maxQueueSize); config = udpConfig; } else