Add UTF8 BOM support (#37)

Author: huksley

README.md

@@ -2,9 +2,11 @@
 
 Based on the syslog4j library bundled with Graylog.
 
-This plugin allows you to forward messages from a Graylog 2.X server in syslog format. Messages can be dispatched over TCP or UDP and formatted as plain text (classic), structured syslog (rfc 5424) or CEF (experimental).
+This plugin allows you to forward messages from a Graylog server in syslog format. 
+Messages can be dispatched over TCP or UDP and formatted as plain text (classic), structured syslog (rfc 5424) or CEF (experimental).
 
-This plugin supports Graylog 2.4.x, 2.5.x, 3.0.0. Other releases might work, pls try to use latest plugin.
+This plugin supports Graylog 2.4.x, 2.5.x, 3.0.0 and 3.3.x. 
+Newever releases might work, please try to use the latest plugin.
 
 ## Graylog marketplace
 
@@ -31,7 +33,7 @@ You can build a plugin (JAR) with `mvn package`.
 - _Syslog port_: Syslog receiver port on remote host, usually 514
 - _Format_: Specify one of plain, structured, full, cef or custom:FQCN (see below for explanation on values)
 
-![Screenshot of add new output dialog](graylog2-output-syslog-2.1.3-parameters.png)
+![Screenshot of add new output dialog](graylog2-output-syslog-parameters.png)
 
 ## Supported formats
 
@@ -145,6 +147,20 @@ If existing fields does not contain such keys, following fields will be added to
 | msg          | Message text (`message`)                     |
 | externalId   | Message ID (assigned by Graylog)             |
 
+### Receiving and sending UTF-8 messages.
+
+Graylog internally are fully UTF-8 capable. All messages are stored as Unicode. When sending messages to syslog server, 
+
+RFC requires adding BOM mark to the messages to identify the string is UTF-8 encoded.
+
+```
+curl -XPOST http://localhost:12201/gelf -p0 -d '{"short_message":"Hello there🙂Ё Ђ Ѓ Є Ѕ І Ї Ј Љ Њ Ћ Ќ Ў Џ А Б В Г Д Е Ж З И Й К Л М Н О П Р С Т У Ф Х Ц Ч Ш Щ Ъ Ы Ь Э Ю Я а б в г д е ж з и й к л м н о п р с т у ф х ц ч ш щ ъ ы ь э ю я À Á Â Ã Ä Å Æ Ç È É Ê Ë Ì Í Î Ï Ð Ñ Ò Ó Ô Õ Ö × Ø Ù Ú Û Ü Ý Þ ß à á â ã ä å æ ç è é ê ë ì í î ï ð ñ ò ó ô õ ö ÷ ø ù ú û ü ý þ ÿ "}'
+```
+
+When running syslog (syslog-ng v 3.27.1), this gets written to /var/log/messages
+
+Jun 20 19:31:43 Ruslans-MacBook-Pro11111111111111111111111 .local Hello there🙂Ё Ђ Ѓ Є Ѕ І Ї Ј Љ Њ Ћ Ќ Ў Џ А Б В Г Д Е Ж З И Й К Л М Н О П Р С Т У Ф Х Ц Ч Ш Щ Ъ Ы Ь Э Ю Я а б в г д е ж з и й к л м н о п р с т у ф х ц ч ш щ ъ ы ь э ю я À Á Â Ã Ä Å Æ Ç È É Ê Ë Ì Í Î Ï Ð Ñ Ò Ó Ô Õ Ö × Ø Ù Ú Û Ü Ý Þ ß à á â ã ä å æ ç è é ê ë ì í î ï ð ñ ò ó ô õ ö ÷ ø ù ú û ü ý þ ÿ
+
 ## Links
 
 - https://tools.ietf.org/html/rfc5424

graylog2-output-syslog-2.1.3-parameters.png

@@ -21,5 +21,5 @@ sleep 5
 $GL/bin/graylogctl run
 
 ## Run two consoles additionally:
-## sudo docker run -it -p 514:514/udp -p 601:601 --name syslog-ng balabit/syslog-ng:latest
+## docker run -it -p 514:514/udp -p 514:514/tcp  -p 601:601 --name syslog-ng balabit/syslog-ng:latest
 ## docker exec syslog-ng tail -f /var/log/messages

graylog2-output-syslog-parameters.png

@@ -16,6 +16,7 @@
 import org.graylog2.syslog4j.SyslogConfigIF;
 import org.graylog2.syslog4j.SyslogIF;
 import org.graylog2.syslog4j.impl.message.processor.SyslogMessageProcessor;
+import org.graylog2.syslog4j.impl.message.processor.structured.StructuredSyslogMessageProcessor;
 import org.graylog2.syslog4j.impl.net.tcp.TCPNetSyslogConfig;
 import org.graylog2.syslog4j.impl.net.tcp.ssl.SSLTCPNetSyslogConfig;
 import org.graylog2.syslog4j.impl.net.udp.UDPNetSyslogConfig;
@@ -38,6 +39,7 @@ public class SyslogOutput implements MessageOutput {
 
 	public final static int PORT_MIN = 9000;
 	public final static int PORT_MAX = 9099;
+	public final static byte[] BOM = { (byte) 0xEF, (byte) 0xBB, (byte) 0xBF };
 
 	private Logger log = Logger.getLogger(SyslogOutput.class.getName());
 	private String host;
@@ -128,6 +130,7 @@ public SyslogOutput(@Assisted Stream stream, @Assisted Configuration conf) {
 		} else {
 			throw new IllegalArgumentException("Unknown protocol: " + protocol);
 		}
+
 		config.setHost(host);
 		config.setPort(port);
 		int maxlen = 16 * 1024;
@@ -142,6 +145,29 @@ public SyslogOutput(@Assisted Stream stream, @Assisted Configuration conf) {
 		String hash = protocol + "_" + host + "_" + port + "_" + format;
 		syslog = Syslog.exists(hash) ? Syslog.getInstance(hash) : Syslog.createInstance(hash, config);
 
+		boolean utf8 = conf.getBoolean("utf8");
+		if (utf8) {
+			syslog.setMessageProcessor(new SyslogMessageProcessor() {
+				public byte[] createPacketData(byte[] header, byte[] message, int start, int length, byte[] splitBeginText, byte[] splitEndText) {
+					byte[] buf = super.createPacketData(header, message, start, length, splitBeginText, splitEndText);
+					byte[] newBuf = new byte[buf.length + BOM.length];
+					System.arraycopy(BOM, 0, newBuf, 0, BOM.length);
+					System.arraycopy(buf, 0, newBuf, BOM.length, buf.length);
+					return newBuf;
+				}
+			});
+
+			syslog.setStructuredMessageProcessor(new StructuredSyslogMessageProcessor() {
+				public byte[] createPacketData(byte[] header, byte[] message, int start, int length, byte[] splitBeginText, byte[] splitEndText) {
+					byte[] buf = super.createPacketData(header, message, start, length, splitBeginText, splitEndText);
+					byte[] newBuf = new byte[buf.length + BOM.length];
+					System.arraycopy(BOM, 0, newBuf, 0, BOM.length);
+					System.arraycopy(buf, 0, newBuf, BOM.length, buf.length);
+					return newBuf;
+				}
+			});
+		}
+
 		sender = createSender(format, conf);
 
 		if (sender instanceof TransparentSyslogSender) {
@@ -293,6 +319,7 @@ public ConfigurationRequest getRequestedConfiguration() {
 			configurationRequest.addField(new TextField("truststore", "Trust store", "", "Path to Java keystore (required for SSL over TCP). Optional (if not set, equals to key store). Must contain peers we trust connecting to.", ConfigurationField.Optional.OPTIONAL));
 			configurationRequest.addField(new TextField("truststorePassword", "Trust store password", "", "", ConfigurationField.Optional.OPTIONAL));
 
+			configurationRequest.addField(new BooleanField("utf8", "UTF-8 BOM", false,"Always add BOM to messages send. Use this to conform to RFC 5424 requirements for UTF-8 messages."));
 			return configurationRequest;
 		}
 	}