Support for Http Proxy Headers (#119)

* Add logging of x-forwarded-* and x-custom-host http headers

Users can choose to log the following for http headers in the request logs:

* x-forwarded-for
* x-forwarded-host
* x-forwarded-proto
* x-custom-host

This requires the environment variable `LOG_SENSITIVE_CONNECTION_DATA` to be set to `true`.
If there are no headers, no fields will be added to the request logs. If there are headers, but
`LOG_SENSITIVE_CONNECTION_DATA` is set to `false`(default) the values will be logged as 'redacted'.

* Log SSL connection headers

HA-Proxy can add certain headers for terminated ssl connections [1].
These headers can be added to the request logs if environment variable
LOG_SSL_HEADERS is set to be true. Otherwise no fields will be added
to the request logs.

[1] https://www.haproxy.com/blog/ssl-client-certificate-information-in-http-headers-and-logs/

Co-authored-by: Christian Dinse <[email protected]>

Author: KarstenSchnitter

cf-java-logging-support-core/beats/request-metrics/docs/fields.asciidoc

@@ -695,3 +695,132 @@ client, followed by proxy server addresses that forwarded the client
 request.
 
 
+==== x_forwarded_host
+
+type: string
+
+example: requested-host.example.com
+
+required: False
+
+The originally requested host by the client in the host HTTP request header.
+A proxy may forward the host header using the x-forwarded-host header.
+
+
+==== x_forwarded_proto
+
+type: string
+
+example: https
+
+required: False
+
+The original protocol used by the client to connect to the proxy or load balancer
+before the application.
+
+
+==== x_custom_host
+
+type: string
+
+example: central-host.example.com
+
+required: False
+
+A header set by a proxy or load balancer for special use-cases.
+
+
+==== x_ssl_client
+
+type: string
+
+example: 0
+
+required: False
+
+A header set by HA-Proxy to indicate usage of a secured connection by the 
+client (1) or not (0).
+
+
+==== x_ssl_client_verify
+
+type: string
+
+example: 0
+
+required: False
+
+A header set by HA-Proxy to indicate the status code of the TLS/SSL connection.
+
+
+==== x_ssl_client_subject_dn
+
+type: string
+
+example: /C=FR/ST=Ile de France/L=Jouy en Josas/O=haproxy.com/CN=client1/[email protected]
+
+required: False
+
+A header set by HA-Proxy to provide the full distinguished name of the
+client certificate.
+
+
+==== x_ssl_client_subject_cn
+
+type: string
+
+example: client1
+
+required: False
+
+A header set by HA-Proxy to provide the full common name of the
+client certificate.
+
+
+==== x_ssl_client_issuer_dn
+
+type: string
+
+example: /C=FR/ST=Ile de France/L=Jouy en Josas/O=haproxy.com/CN=haproxy.com/[email protected]
+
+required: False
+
+A header set by HA-Proxy to provide the full distinguished name of the
+issuer of the client certificate.
+
+
+==== x_ssl_client_notbefore
+
+type: string
+
+example: 130613144555Z
+
+required: False
+
+A header set by HA-Proxy to provide the start date of the client
+certificate as a formatted string YYMMDDhhmmss.
+
+
+==== x_ssl_client_notafter
+
+type: string
+
+example: 140613144555Z
+
+required: False
+
+A header set by HA-Proxy to provide the end date of the client
+certificate as a formatted string YYMMDDhhmmss.
+
+
+==== x_ssl_client_session_id
+
+type: string
+
+example: session-id
+
+required: False
+
+A header to indicate the SSL client session id.
+
+

cf-java-logging-support-core/beats/request-metrics/etc/fields.yml

@@ -501,3 +501,88 @@ request-metrics:
         Comma-separated list of IP addresses, the left-most being the original
         client, followed by proxy server addresses that forwarded the client
         request.
+
+    - name: "x_forwarded_host"
+      type: string
+      required: false
+      example: "requested-host.example.com"
+      description: |
+        The originally requested host by the client in the host HTTP request header.
+        A proxy may forward the host header using the x-forwarded-host header.
+
+    - name: "x_forwarded_proto"
+      type: string
+      required: false
+      example: "https"
+      description: |
+        The original protocol used by the client to connect to the proxy or load balancer
+        before the application.
+
+    - name: "x_custom_host"
+      type: string
+      required: false
+      example: "central-host.example.com"
+      description: |
+        A header set by a proxy or load balancer for special use-cases.
+
+    - name: "x_ssl_client"
+      type: string
+      required: false
+      example: "0"
+      description: |
+        A header set by HA-Proxy to indicate usage of a secured connection by the 
+        client (1) or not (0).
+
+    - name: "x_ssl_client_verify"
+      type: string
+      required: false
+      example: "0"
+      description: |
+        A header set by HA-Proxy to indicate the status code of the TLS/SSL connection.
+
+    - name: "x_ssl_client_subject_dn"
+      type: string
+      required: false
+      example: "/C=FR/ST=Ile de France/L=Jouy en Josas/O=haproxy.com/CN=client1/[email protected]"
+      description: |
+        A header set by HA-Proxy to provide the full distinguished name of the
+        client certificate.
+
+    - name: "x_ssl_client_subject_cn"
+      type: string
+      required: false
+      example: "client1"
+      description: |
+        A header set by HA-Proxy to provide the full common name of the
+        client certificate.
+
+    - name: "x_ssl_client_issuer_dn"
+      type: string
+      required: false
+      example: "/C=FR/ST=Ile de France/L=Jouy en Josas/O=haproxy.com/CN=haproxy.com/[email protected]"
+      description: |
+        A header set by HA-Proxy to provide the full distinguished name of the
+        issuer of the client certificate.
+
+    - name: "x_ssl_client_notbefore"
+      type: string
+      required: false
+      example: "130613144555Z"
+      description: |
+        A header set by HA-Proxy to provide the start date of the client
+        certificate as a formatted string YYMMDDhhmmss.
+
+    - name: "x_ssl_client_notafter"
+      type: string
+      required: false
+      example: "140613144555Z"
+      description: |
+        A header set by HA-Proxy to provide the end date of the client
+        certificate as a formatted string YYMMDDhhmmss.
+
+    - name: "x_ssl_client_session_id"
+      type: string
+      required: false
+      example: "session-id"
+      description: |
+        A header to indicate the SSL client session id.

cf-java-logging-support-core/beats/request-metrics/etc/request-metrics.template.json

@@ -273,10 +273,65 @@
           "ignore_malformed": true,
           "type": "long"
         },
+        "x_custom_host": {
+          "doc_values": true,
+          "index": "not_analyzed",
+          "type": "string"
+        },
         "x_forwarded_for": {
           "doc_values": true,
           "index": "not_analyzed",
           "type": "string"
+        },
+        "x_forwarded_host": {
+          "doc_values": true,
+          "index": "not_analyzed",
+          "type": "string"
+        },
+        "x_forwarded_proto": {
+          "doc_values": true,
+          "index": "not_analyzed",
+          "type": "string"
+        },
+        "x_ssl_client": {
+          "doc_values": true,
+          "index": "not_analyzed",
+          "type": "string"
+        },
+        "x_ssl_client_issuer_dn": {
+          "doc_values": true,
+          "index": "not_analyzed",
+          "type": "string"
+        },
+        "x_ssl_client_notafter": {
+          "doc_values": true,
+          "index": "not_analyzed",
+          "type": "string"
+        },
+        "x_ssl_client_notbefore": {
+          "doc_values": true,
+          "index": "not_analyzed",
+          "type": "string"
+        },
+        "x_ssl_client_session_id": {
+          "doc_values": true,
+          "index": "not_analyzed",
+          "type": "string"
+        },
+        "x_ssl_client_subject_cn": {
+          "doc_values": true,
+          "index": "not_analyzed",
+          "type": "string"
+        },
+        "x_ssl_client_subject_dn": {
+          "doc_values": true,
+          "index": "not_analyzed",
+          "type": "string"
+        },
+        "x_ssl_client_verify": {
+          "doc_values": true,
+          "index": "not_analyzed",
+          "type": "string"
         }
       }
     }

cf-java-logging-support-core/src/main/java/com/sap/hcp/cf/logging/common/Fields.java

@@ -65,4 +65,15 @@ public interface Fields {
   public String RESPONSE_CONTENT_TYPE = "response_content_type";
   public String REFERER = "referer";
   public String X_FORWARDED_FOR = "x_forwarded_for";
+  public String X_FORWARDED_HOST = "x_forwarded_host";
+  public String X_FORWARDED_PROTO = "x_forwarded_proto";
+  public String X_CUSTOM_HOST = "x_custom_host";
+  public String X_SSL_CLIENT = "x_ssl_client";
+  public String X_SSL_CLIENT_VERIFY = "x_ssl_client_verify";
+  public String X_SSL_CLIENT_SUBJECT_DN = "x_ssl_client_subject_dn";
+  public String X_SSL_CLIENT_SUBJECT_CN = "x_ssl_client_subject_cn";
+  public String X_SSL_CLIENT_ISSUER_DN = "x_ssl_client_issuer_dn";
+  public String X_SSL_CLIENT_NOTBEFORE = "x_ssl_client_notbefore";
+  public String X_SSL_CLIENT_NOTAFTER = "x_ssl_client_notafter";
+  public String X_SSL_CLIENT_SESSION_ID = "x_ssl_client_session_id";
 }

cf-java-logging-support-core/src/main/java/com/sap/hcp/cf/logging/common/LogOptionalFieldsSettings.java

@@ -12,6 +12,7 @@ public class LogOptionalFieldsSettings {
     private final boolean logSensitiveConnectionData;
     private final boolean logRemoteUserField;
     private final boolean logRefererField;
+    private final boolean logSslHeaders;
 
     public LogOptionalFieldsSettings(String invokingClass) {
         this(new Environment(), invokingClass);
@@ -22,6 +23,7 @@ public LogOptionalFieldsSettings(String invokingClass) {
                                                              invokingClass);
         logRemoteUserField = readEnvironmentVariable(Environment.LOG_REMOTE_USER, environment, invokingClass);
         logRefererField = readEnvironmentVariable(Environment.LOG_REFERER, environment, invokingClass);
+        logSslHeaders = readEnvironmentVariable(Environment.LOG_SSL_HEADERS, environment, invokingClass);
     }
 
     private static boolean readEnvironmentVariable(String environmentVariableKey, Environment environment,
@@ -63,4 +65,8 @@ public boolean isLogRemoteUserField() {
     public boolean isLogRefererField() {
         return logRefererField;
     }
+
+    public boolean isLogSslHeaders() {
+        return logSslHeaders;
+    }
 }

cf-java-logging-support-core/src/main/java/com/sap/hcp/cf/logging/common/helper/Environment.java

@@ -5,6 +5,7 @@ public class Environment {
     public static final String LOG_SENSITIVE_CONNECTION_DATA = "LOG_SENSITIVE_CONNECTION_DATA";
     public static final String LOG_REMOTE_USER = "LOG_REMOTE_USER";
     public static final String LOG_REFERER = "LOG_REFERER";
+    public static final String LOG_SSL_HEADERS = "LOG_SSL_HEADERS";
 
     public String getVariable(String name) {
         return System.getenv(name);

cf-java-logging-support-core/src/main/java/com/sap/hcp/cf/logging/common/request/HttpHeaders.java

@@ -15,15 +15,36 @@ public enum HttpHeaders implements HttpHeader {
                                                CONTENT_LENGTH("content-length"), //
                                                CONTENT_TYPE("content-type"), //
                                                REFERER("referer"), //
-                                               X_FORWARDED_FOR("x-forwarded-for"), //
+                                               X_CUSTOM_HOST("x-custom-host", Fields.X_CUSTOM_HOST), //
+                                               X_FORWARDED_FOR("x-forwarded-for", Fields.X_FORWARDED_FOR), //
+                                               X_FORWARDED_HOST("x-forwarded-host", Fields.X_FORWARDED_HOST), //
+                                               X_FORWARDED_PROTO("x-forwarded-proto", Fields.X_FORWARDED_PROTO), //
+                                               X_SSL_CLIENT("x-ssl-client", Fields.X_SSL_CLIENT), //
+                                               X_SSL_CLIENT_VERIFY("x-ssl-client-verify", Fields.X_SSL_CLIENT_VERIFY), //
+                                               X_SSL_CLIENT_SUBJECT_DN("x-ssl-client-subject-dn",
+                                                                       Fields.X_SSL_CLIENT_SUBJECT_DN), //
+                                               X_SSL_CLIENT_SUBJECT_CN("x-ssl-client-subject-cn",
+                                                                       Fields.X_SSL_CLIENT_SUBJECT_CN), //
+                                               X_SSL_CLIENT_ISSUER_DN("x-ssl-client-issuer-dn",
+                                                                      Fields.X_SSL_CLIENT_ISSUER_DN), //
+                                               X_SSL_CLIENT_NOTBEFORE("x-ssl-client-notbefore",
+                                                                      Fields.X_SSL_CLIENT_NOTBEFORE), //
+                                               X_SSL_CLIENT_NOTAFTER("x-ssl-client-notafter",
+                                                                     Fields.X_SSL_CLIENT_NOTAFTER), //
+                                               X_SSL_CLIENT_SESSION_ID("x-ssl-client-session-id",
+                                                                       Fields.X_SSL_CLIENT_SESSION_ID), //
                                                X_VCAP_REQUEST_ID("x-vcap-request-id", Fields.REQUEST_ID, true), //
                                                CORRELATION_ID("X-CorrelationID", Fields.CORRELATION_ID, true,
                                                               X_VCAP_REQUEST_ID), //
                                                SAP_PASSPORT("sap-passport", Fields.SAP_PASSPORT, true), //
                                                TENANT_ID("tenantid", Fields.TENANT_ID, true); //
 
     private HttpHeaders(String name) {
-        this(name, null, false);
+        this(name, null);
+    }
+
+    private HttpHeaders(String name, String field) {
+        this(name, field, false);
     }
 
     private HttpHeaders(String name, String field, boolean isPropagated, HttpHeaders... aliases) {