[client] Add JSON format and timestamp for logs (#208)

* [client] Add JSON format and timestamp for logs

* [client] modifications regarding comments

Author: axelfahy

pycti/api/opencti_api_client.py

@@ -1,12 +1,12 @@
 # coding: utf-8
-
 import datetime
 import io
 import json
 import logging
 from typing import Union
 
 import magic
+from pythonjsonlogger import jsonlogger
 import requests
 import urllib3
 
@@ -50,6 +50,19 @@
 urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
 
 
+class CustomJsonFormatter(jsonlogger.JsonFormatter):
+    def add_fields(self, log_record, record, message_dict):
+        super(CustomJsonFormatter, self).add_fields(log_record, record, message_dict)
+        if not log_record.get("timestamp"):
+            # This doesn't use record.created, so it is slightly off
+            now = datetime.datetime.utcnow().strftime("%Y-%m-%dT%H:%M:%S.%fZ")
+            log_record["timestamp"] = now
+        if log_record.get("level"):
+            log_record["level"] = log_record["level"].upper()
+        else:
+            log_record["level"] = record.levelname
+
+
 class File:
     def __init__(self, name, data, mime="text/plain"):
         self.name = name
@@ -76,9 +89,19 @@ class OpenCTIApiClient:
             "https: "http://my_proxy:8080"
         }
         ```
+    :param json_logging: format the logs as json if set to True
+    :type json_logging: bool, optional
     """
 
-    def __init__(self, url, token, log_level="info", ssl_verify=False, proxies={}):
+    def __init__(
+        self,
+        url,
+        token,
+        log_level="info",
+        ssl_verify=False,
+        proxies={},
+        json_logging=False,
+    ):
         """Constructor method"""
 
         # Check configuration
@@ -94,7 +117,17 @@ def __init__(self, url, token, log_level="info", ssl_verify=False, proxies={}):
         numeric_level = getattr(logging, self.log_level.upper(), None)
         if not isinstance(numeric_level, int):
             raise ValueError("Invalid log level: " + self.log_level)
-        logging.basicConfig(level=numeric_level)
+
+        if json_logging:
+            log_handler = logging.StreamHandler()
+            log_handler.setLevel(self.log_level.upper())
+            formatter = CustomJsonFormatter(
+                "%(timestamp)s %(level)s %(name)s %(message)s"
+            )
+            log_handler.setFormatter(formatter)
+            logging.basicConfig(handlers=[log_handler], level=numeric_level, force=True)
+        else:
+            logging.basicConfig(level=numeric_level)
 
         # Define API
         self.api_token = token

pycti/connector/opencti_connector_helper.py

@@ -381,6 +381,9 @@ def __init__(self, config: Dict) -> None:
         self.opencti_ssl_verify = get_config_variable(
             "OPENCTI_SSL_VERIFY", ["opencti", "ssl_verify"], config, False, True
         )
+        self.opencti_json_logging = get_config_variable(
+            "OPENCTI_JSON_LOGGING", ["opencti", "json_logging"], config
+        )
         # Load connector config
         self.connect_id = get_config_variable(
             "CONNECTOR_ID", ["connector", "id"], config
@@ -438,7 +441,10 @@ def __init__(self, config: Dict) -> None:
 
         # Initialize configuration
         self.api = OpenCTIApiClient(
-            self.opencti_url, self.opencti_token, self.log_level
+            self.opencti_url,
+            self.opencti_token,
+            self.log_level,
+            json_logging=self.opencti_json_logging,
         )
         # Register the connector in OpenCTI
         self.connector = OpenCTIConnector(

requirements.txt

@@ -5,5 +5,6 @@ datefinder~=0.7
 stix2~=2.1.0
 pika~=1.2
 sseclient~=0.0.27
+python_json_logger~=2.0.2
 python-magic~=0.4.24; sys_platform == 'linux' or sys_platform == 'darwin'
 python-magic-bin~=0.4.14; sys_platform == 'win32'

tests/cases/connectors.py

@@ -60,6 +60,7 @@ def __init__(self, config_file_path: str, api_client: OpenCTIApiClient, data: Di
         os.environ["OPENCTI_URL"] = api_client.api_url
         os.environ["OPENCTI_TOKEN"] = api_client.api_token
         os.environ["OPENCTI_SSL_VERIFY"] = str(api_client.ssl_verify)
+        os.environ["OPENCTI_JSON_LOGGING"] = "true"
 
         config = (
             yaml.load(open(config_file_path), Loader=yaml.FullLoader)