feat: add capability to customize extraction of request, response info

#68

Author: Bob Bui

CHANGELOG.md

@@ -4,6 +4,9 @@ All notable changes to this project will be documented in this file.
 This project adheres to [Semantic Versioning](http://semver.org/).  
 The format is based on [Keep a Changelog](http://keepachangelog.com/).
 
+## 1.4.0rc - 2021-04-09
+ - add capability to customize extraction of request, response info #68
+
 ## 1.3.0 - 2021-03-25
  - add fastapi support #65
 

README.md

@@ -214,6 +214,7 @@ json_logging.config_root_logger()
 Customer JSON log formatter can be passed to init method. see examples for more detail: [non web](https://github.com/thangbn/json-logging-python/blob/master/example/custom_log_format.py), 
 [web](https://github.com/thangbn/json-logging-python/blob/master/example/custom_log_format_request.py)
 
+Custom extraction of request and response can be done by subclass **json_logging.RequestResponseDataExtractionBase**
 
 
 ## 2.7 Exclude certain URl from request instrumentation

example/custom_log_format_request.py

@@ -14,6 +14,7 @@ class CustomRequestJSONLog(json_logging.JSONLogWebFormatter):
 
     def format(self, record):
         json_customized_log_object = ({
+            "type": "request",
             "customized_prop": "customized value",
             "correlation_id": json_logging.get_correlation_id(),
         })

json_logging/__init__.py

@@ -98,6 +98,46 @@ def init_non_web(*args, **kw):
     __init(*args, **kw)
 
 
+class RequestResponseDataExtractorBase(dict):
+    """
+        class that keep HTTP request information for request instrumentation logging
+    """
+
+    def __init__(self, request, **kwargs):
+        super(RequestResponseDataExtractorBase, self).__init__(**kwargs)
+
+    def update_response_status(self, response):
+        pass
+
+
+class RequestResponseDataExtractor(RequestResponseDataExtractorBase):
+    """
+        default implementation
+    """
+
+    def __init__(self, request, **kwargs):
+        super(RequestResponseDataExtractor, self).__init__(request, **kwargs)
+        utcnow = datetime.utcnow()
+        self.request_start = utcnow
+        self.request = request
+        self.request_received_at = util.iso_time_format(utcnow)
+
+    # noinspection PyAttributeOutsideInit
+    def update_response_status(self, response):
+        """
+        update response information into this object, must be called before invoke request logging statement
+        :param response:
+        """
+        response_adapter = _request_util.response_adapter
+        utcnow = datetime.utcnow()
+        time_delta = utcnow - self.request_start
+        self.response_time_ms = int(time_delta.total_seconds()) * 1000 + int(time_delta.microseconds / 1000)
+        self.response_status = response_adapter.get_status_code(response)
+        self.response_size_b = response_adapter.get_response_size(response)
+        self.response_content_type = response_adapter.get_content_type(response)
+        self.response_sent_at = util.iso_time_format(utcnow)
+
+
 def __init(framework_name=None, custom_formatter=None, enable_json=False):
     """
     Initialize JSON logging support, if no **framework_name** passed, logging will be initialized in non-web context.
@@ -153,14 +193,16 @@ def __init(framework_name=None, custom_formatter=None, enable_json=False):
     util.update_formatter_for_loggers(existing_loggers, _default_formatter)
 
 
-def init_request_instrument(app=None, custom_formatter=None, exclude_url_patterns=[]):
+def init_request_instrument(app=None, custom_formatter=None, exclude_url_patterns=[],
+                            request_response_data_extractor_class=RequestResponseDataExtractor):
     """
     Configure the request instrumentation logging configuration for given web app. Must be called after init method
 
     If **custom_formatter** is passed, it will use this formatter over the default.
 
     :param app: current web application instance
     :param custom_formatter: formatter to override default JSONRequestLogFormatter.
+    :param request_response_data_extractor_class: requestinfo to override default json_logging.RequestInfo.
     """
 
     if _current_framework is None or _current_framework == '-':
@@ -170,8 +212,11 @@ def init_request_instrument(app=None, custom_formatter=None, exclude_url_pattern
         if not issubclass(custom_formatter, logging.Formatter):
             raise ValueError('custom_formatter is not subclass of logging.Formatter', custom_formatter)
 
+    if not issubclass(request_response_data_extractor_class, RequestResponseDataExtractorBase):
+        raise ValueError('request_response_data_extractor_class is not subclass of json_logging.RequestInfoBase', custom_formatter)
+
     configurator = _current_framework['app_request_instrumentation_configurator']()
-    configurator.config(app, exclude_url_patterns=exclude_url_patterns)
+    configurator.config(app, request_response_data_extractor_class, exclude_url_patterns=exclude_url_patterns)
 
     formatter = custom_formatter if custom_formatter else JSONRequestLogFormatter
     request_logger = configurator.request_logger
@@ -182,8 +227,6 @@ def init_request_instrument(app=None, custom_formatter=None, exclude_url_pattern
 
 
 def get_request_logger():
-    global _request_logger
-
     if _current_framework is None or _current_framework == '-':
         raise RuntimeError(
             "request_logger is only available if json_logging is inited with a web app, "
@@ -196,34 +239,6 @@ def get_request_logger():
     return instance.request_logger
 
 
-class RequestInfo(dict):
-    """
-        class that keep HTTP request information for request instrumentation logging
-    """
-
-    def __init__(self, request, **kwargs):
-        super(RequestInfo, self).__init__(**kwargs)
-        utcnow = datetime.utcnow()
-        self.request_start = utcnow
-        self.request = request
-        self.request_received_at = util.iso_time_format(utcnow)
-
-    # noinspection PyAttributeOutsideInit
-    def update_response_status(self, response):
-        """
-        update response information into this object, must be called before invoke request logging statement
-        :param response:
-        """
-        response_adapter = _request_util.response_adapter
-        utcnow = datetime.utcnow()
-        time_delta = utcnow - self.request_start
-        self.response_time_ms = int(time_delta.total_seconds()) * 1000 + int(time_delta.microseconds / 1000)
-        self.response_status = response_adapter.get_status_code(response)
-        self.response_size_b = response_adapter.get_response_size(response)
-        self.response_content_type = response_adapter.get_content_type(response)
-        self.response_sent_at = util.iso_time_format(utcnow)
-
-
 class BaseJSONFormatter(logging.Formatter):
     """
        Base class for JSON formatters
@@ -406,5 +421,6 @@ def init_connexion(custom_formatter=None, enable_json=False):
                                request_adapter_class=fastapi_support.FastAPIRequestAdapter,
                                response_adapter_class=fastapi_support.FastAPIResponseAdapter)
 
+
 def init_fastapi(custom_formatter=None, enable_json=False):
     __init(framework_name='fastapi', custom_formatter=custom_formatter, enable_json=enable_json)

json_logging/framework/connexion/__init__.py

@@ -31,7 +31,7 @@ def is_connexion_present():
 
 
 class ConnexionAppRequestInstrumentationConfigurator(AppRequestInstrumentationConfigurator):
-    def config(self, app, exclude_url_patterns=[]):
+    def config(self, app, request_response_data_extractor_class, exclude_url_patterns=[]):
         if not is_connexion_present():
             raise RuntimeError("connexion is not available in system runtime")
         from flask.app import Flask
@@ -51,7 +51,7 @@ def config(self, app, exclude_url_patterns=[]):
         @app.app.before_request
         def before_request():
             if is_not_match_any_pattern(_current_request.path, exclude_url_patterns):
-                g.request_info = json_logging.RequestInfo(_current_request)
+                g.request_info = request_response_data_extractor_class(_current_request)
 
         @app.app.after_request
         def after_request(response):

json_logging/framework/fastapi/implementation.py

@@ -15,6 +15,8 @@
 from starlette.responses import Response
 from starlette.types import ASGIApp
 
+_request_config_class = None
+
 
 class JSONLoggingASGIMiddleware(BaseHTTPMiddleware):
     def __init__(self, app: ASGIApp, exclude_url_patterns=tuple()) -> None:
@@ -29,7 +31,7 @@ async def dispatch(self, request: Request, call_next: RequestResponseEndpoint) -
         if not log_request:
             return await call_next(request)
 
-        request_info = json_logging.RequestInfo(request)
+        request_info = _request_config_class(request)
         response = await call_next(request)
         request_info.update_response_status(response)
         self.request_logger.info(
@@ -39,10 +41,13 @@ async def dispatch(self, request: Request, call_next: RequestResponseEndpoint) -
 
 
 class FastAPIAppRequestInstrumentationConfigurator(AppRequestInstrumentationConfigurator):
-    def config(self, app, exclude_url_patterns=tuple()):
+    def config(self, app, request_response_data_extractor_class, exclude_url_patterns=[]):
         if not isinstance(app, fastapi.FastAPI):
             raise RuntimeError("app is not a valid fastapi.FastAPI instance")
 
+        global _request_config_class
+        _request_config_class = request_response_data_extractor_class
+
         # Disable standard logging
         logging.getLogger('uvicorn.access').disabled = True
 

json_logging/framework/flask/__init__.py

@@ -26,7 +26,7 @@ def is_flask_present():
 
 
 class FlaskAppRequestInstrumentationConfigurator(AppRequestInstrumentationConfigurator):
-    def config(self, app, exclude_url_patterns=[]):
+    def config(self, app, request_response_data_extractor_class, exclude_url_patterns=[]):
         if not is_flask_present():
             raise RuntimeError("flask is not available in system runtime")
         from flask.app import Flask
@@ -47,7 +47,7 @@ def config(self, app, exclude_url_patterns=[]):
         @app.before_request
         def before_request():
             if is_not_match_any_pattern(_current_request.path, exclude_url_patterns):
-                g.request_info = json_logging.RequestInfo(_current_request)
+                g.request_info = request_response_data_extractor_class(_current_request)
 
         @app.after_request
         def after_request(response):

json_logging/framework/quart/__init__.py

@@ -27,7 +27,7 @@ def is_quart_present():
 
 
 class QuartAppRequestInstrumentationConfigurator(AppRequestInstrumentationConfigurator):
-    def config(self, app, exclude_url_patterns=[]):
+    def config(self, app, request_response_data_extractor_class, exclude_url_patterns=[]):
         if not is_quart_present():
             raise RuntimeError("quart is not available in system runtime")
         from quart.app import Quart
@@ -51,7 +51,7 @@ def config(self, app, exclude_url_patterns=[]):
         @app.before_request
         def before_request():
             if is_not_match_any_pattern(_current_request.path, exclude_url_patterns):
-                g.request_info = json_logging.RequestInfo(_current_request)
+                g.request_info = request_response_data_extractor_class(_current_request)
 
         @app.after_request
         def after_request(response):

json_logging/framework/sanic/__init__.py

@@ -50,7 +50,7 @@ def config(self):
 
 
 class SanicAppRequestInstrumentationConfigurator(AppRequestInstrumentationConfigurator):
-    def config(self, app, exclude_url_patterns=[]):
+    def config(self, app, request_response_data_extractor_class, exclude_url_patterns=[]):
         if not is_sanic_present():
             raise RuntimeError("Sanic is not available in system runtime")
         # noinspection PyPackageRequirements
@@ -67,7 +67,7 @@ def config(self, app, exclude_url_patterns=[]):
         @app.middleware("request")
         def before_request(request):
             if is_not_match_any_pattern(request.path, exclude_url_patterns):
-                request.ctx.request_info = json_logging.RequestInfo(request)
+                request.ctx.request_info = request_response_data_extractor_class(request)
 
         @app.middleware("response")
         def after_request(request, response):
@@ -78,9 +78,6 @@ def after_request(request, response):
                     "", extra={"request_info": request_info, "type": "request"}
                 )
 
-    def get_request_logger(self):
-        return self.request_logger
-
 
 class SanicRequestAdapter(RequestAdapter):
     @staticmethod

json_logging/framework_base.py

@@ -189,11 +189,13 @@ def __new__(cls, *args, **kw):
             cls._instance.request_logger = None
         return cls._instance
 
-    def config(self, app, exclude_url_patterns=None):
+    def config(self, app, request_response_data_extractor_class, exclude_url_patterns=[]):
         """
         configuration logic
 
-        :param app:
+        :param app: web application object instance
+        :param request_response_data_extractor_class: request info class
+        :param exclude_url_patterns: list of URL regex pattern to be excluded from request logging
         """
         raise NotImplementedError