Added support for log categories for application logs (via Marker)

Jersey HTTP header propagation based on @context HttpHeaders

Author: altenhof

cf-java-logging-support-core/beats/app-logs/docs/fields.asciidoc

@@ -303,6 +303,15 @@ Stacktrace information in case the message is reporting an exception.
 NOTE: This MUST NOT contain NEWLINE or TAB characters!
 
 
+==== categories
+
+type: string
+
+required: False
+
+A list of names to further categorize this log message.
+
+
 === custom_fields Fields
 
 A JSON object representing the custom arguments passed to the logging

cf-java-logging-support-core/beats/app-logs/etc/app-logs.template.json

@@ -18,6 +18,11 @@
         }
       ],
       "properties": {
+        "categories": {
+          "doc_values": true,
+          "index": "not_analyzed",
+          "type": "string"
+        },
         "component_id": {
           "doc_values": true,
           "index": "not_analyzed",

cf-java-logging-support-core/beats/app-logs/etc/fields.yml

@@ -226,6 +226,12 @@ app-logs:
 
         NOTE: This MUST NOT contain NEWLINE or TAB characters!
 
+    - name: "categories"
+      type: string
+      required: false
+      description: |
+        A list of names to further categorize this log message.
+
     - name: "custom_fields"
       type: group
       required: false

cf-java-logging-support-core/src/main/java/com/sap/hcp/cf/logging/common/Fields.java

@@ -28,6 +28,7 @@ public interface Fields {
   public String LEVEL = "level";
   public String MSG = "msg";
   public String STACKTRACE = "stacktrace";
+  public String CATEGORIES = "categories";
   public String CUSTOM_FIELDS = "custom_fields";
   public String REQUEST_ID = "request_id";
   public String REQUEST = "request";

cf-java-logging-support-jersey/src/main/java/com/sap/hcp/cf/logging/jersey/filter/ClientRequestUtils.java

@@ -9,9 +9,9 @@
 
 public class ClientRequestUtils {
 
-	public static Invocation.Builder propagate(Invocation.Builder builder, ContainerRequest req) {
+	public static Invocation.Builder propagate(Invocation.Builder builder, javax.ws.rs.core.HttpHeaders reqHeaders) {
 		if (LogContext.getCorrelationId() == null) {
-			LogContext.initializeContext(req != null ? req.getHeaderString(HttpHeaders.CORRELATION_ID) : null);
+			LogContext.initializeContext(reqHeaders != null ? reqHeaders.getHeaderString(HttpHeaders.CORRELATION_ID) : null);
 		}
 		builder.header(HttpHeaders.CORRELATION_ID, LogContext.getCorrelationId());
 		return builder;

cf-java-logging-support-jersey/src/test/java/com/sap/hcp/cf/logging/jersey/filter/TestChainedResource.java

@@ -6,6 +6,7 @@
 import javax.ws.rs.client.ClientBuilder;
 import javax.ws.rs.client.WebTarget;
 import javax.ws.rs.core.Context;
+import javax.ws.rs.core.HttpHeaders;
 import javax.ws.rs.core.Response;
 import javax.ws.rs.core.UriInfo;
 
@@ -21,7 +22,7 @@
 public class TestChainedResource {
 
 	@GET
-	public Response  getHello(@Context UriInfo ui, @Context ContainerRequest req) {
+	public Response  getHello(@Context UriInfo ui, @Context HttpHeaders reqHeaders) {
 
 		ClientConfig cfg = new ClientConfig();
 		cfg.register(RequestMetricsClientRequestFilter.class);
@@ -32,7 +33,7 @@ public Response  getHello(@Context UriInfo ui, @Context ContainerRequest req) {
 		/*
 		 * -- As this may get executed in a different thread, propagate 
 		 */
-		return ClientRequestUtils.propagate(wt.request(), req).get();		       
+		return ClientRequestUtils.propagate(wt.request(), reqHeaders).get();
 	}
 
 }

cf-java-logging-support-log4j2/src/main/java/com/sap/hcp/cf/log4j2/converter/CategoriesConverter.java

@@ -0,0 +1,56 @@
+package com.sap.hcp.cf.log4j2.converter;
+
+import com.fasterxml.jackson.jr.ob.JSON;
+import com.fasterxml.jackson.jr.ob.JSONComposer;
+import com.fasterxml.jackson.jr.ob.comp.ArrayComposer;
+import org.apache.logging.log4j.Marker;
+import org.apache.logging.log4j.core.LogEvent;
+import org.apache.logging.log4j.core.config.plugins.Plugin;
+import org.apache.logging.log4j.core.pattern.ConverterKeys;
+import org.apache.logging.log4j.core.pattern.LogEventPatternConverter;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.Iterator;
+
+@Plugin(name="CategoriesConverter", category="Converter")
+@ConverterKeys({"categories"})
+public class CategoriesConverter extends LogEventPatternConverter  {
+
+	public static final String WORD = "categories";
+
+	public CategoriesConverter(String[] options) {
+		super(WORD, WORD);
+	}
+	
+	public static CategoriesConverter newInstance(final String[] options) {
+		return new CategoriesConverter(options);
+	}
+	
+	@Override
+	public void format(LogEvent event, StringBuilder toAppendTo) {
+		getMarkers(event.getMarker(), toAppendTo);
+	}
+
+	private void getMarkers(Marker marker, StringBuilder toAppendTo) {
+		try {
+			ArrayComposer<JSONComposer<String>> ac = JSON.std.composeString().startArray();
+			getMarkersRecursively(marker, ac);
+			toAppendTo.append(ac.end().finish());
+		} catch (IOException ex) {
+			LoggerFactory.getLogger(CategoriesConverter.class).error("conversion failed", ex);
+		}
+	}
+
+	private void getMarkersRecursively(Marker marker, ArrayComposer<JSONComposer<String>> ac) throws IOException {
+		if (marker != null) {
+			ac.add(marker.getName());
+			Marker[] parents = marker.getParents();
+			if (parents != null) {
+				for (Marker parent : parents) {
+					getMarkersRecursively(parent, ac);
+				}
+			}
+		}
+	}
+}

cf-java-logging-support-log4j2/src/main/java/com/sap/hcp/cf/log4j2/converter/TimestampConverter.java

@@ -0,0 +1,27 @@
+package com.sap.hcp.cf.log4j2.converter;
+
+import org.apache.logging.log4j.core.LogEvent;
+import org.apache.logging.log4j.core.config.plugins.Plugin;
+import org.apache.logging.log4j.core.pattern.ConverterKeys;
+import org.apache.logging.log4j.core.pattern.LogEventPatternConverter;
+
+@Plugin(name="TimestampConverter", category="Converter")
+@ConverterKeys({"tstamp"})
+public class TimestampConverter extends LogEventPatternConverter  {
+
+	public static final String WORD = "tstamp";
+
+	public TimestampConverter(String[] options) {
+		super(WORD, WORD);
+	}
+	
+	public static TimestampConverter newInstance(final String[] options) {
+		return new TimestampConverter(options);
+	}
+	
+	@Override
+	public void format(LogEvent event, StringBuilder toAppendTo) {
+		toAppendTo.append(System.nanoTime());
+	}
+
+}

cf-java-logging-support-log4j2/src/main/java/com/sap/hcp/cf/log4j2/layout/LayoutPatterns.java

@@ -24,7 +24,8 @@ public enum PATTERN_KEY {
 	 */
 	private static final String COMMON_PREFIX_PATTERN = 
 		"{ " +
-		JSON_FIELD(Fields.WRITTEN_AT, "%d{yyyy-MM-dd'T'HH:mm:ss.SSSX}{UTC}", true, false) +
+		JSON_FIELD(Fields.WRITTEN_AT, "%d{yyyy-MM-dd'T'HH:mm:ss.SSSX}{UTC}", true, true) +
+		JSON_FIELD(Fields.WRITTEN_TS, "%tstamp", false, false) +
 		"%replace{%ctxp}{(.+)}{,$1},";
 
 	/*
@@ -43,6 +44,7 @@ public enum PATTERN_KEY {
 		JSON_FIELD(Fields.LOGGER, "%logger", true, true) +
 		JSON_FIELD(Fields.THREAD, "%thread", true, true) +
 		JSON_FIELD(Fields.LEVEL, "%p", true, true) +
+		JSON_FIELD(Fields.CATEGORIES, "%categories", false, true) +
 		JSON_FIELD(Fields.MSG, "%jsonmsg{escape}%replace{%args{custom_fields}}{(.+)}{,$1}", false, false);
 
 	/*

cf-java-logging-support-log4j2/src/test/java/com/sap/hcp/cf/logging/common/AbstractTest.java

@@ -2,6 +2,7 @@
 
 import java.io.ByteArrayOutputStream;
 import java.io.PrintStream;
+import java.util.List;
 import java.util.Map;
 
 import org.junit.After;
@@ -43,6 +44,15 @@ protected String getField(String fieldName) {
 		}
 	}
 
+	protected List<String> getList(String fieldName) {
+		try {
+			return (List<String>)JSON.std.mapFrom(getLastLine()).get(fieldName);
+		} catch (Exception ex) {
+			ex.printStackTrace();
+			return null;
+		}
+	}
+
 	private String getLastLine() {
 		String[] lines = this.outContent.toString().split("\n");
 		return lines[lines.length-1];