[client] Global change of log management (#521)

Author: richard-julien

pycti/api/__init__.py

@@ -1,3 +0,0 @@
-import logging
-
-LOGGER = logging.getLogger(__name__)

pycti/api/opencti_api_client.py

@@ -3,16 +3,12 @@
 import datetime
 import io
 import json
-import logging
 from typing import Union
 
 import magic
 import requests
-import urllib3
-from pythonjsonlogger import jsonlogger
 
 from pycti import __version__
-from pycti.api import LOGGER
 from pycti.api.opencti_api_connector import OpenCTIApiConnector
 from pycti.api.opencti_api_playbook import OpenCTIApiPlaybook
 from pycti.api.opencti_api_work import OpenCTIApiWork
@@ -65,24 +61,10 @@
 from pycti.entities.opencti_tool import Tool
 from pycti.entities.opencti_vocabulary import Vocabulary
 from pycti.entities.opencti_vulnerability import Vulnerability
+from pycti.utils.opencti_logger import logger
 from pycti.utils.opencti_stix2 import OpenCTIStix2
 from pycti.utils.opencti_stix2_utils import OpenCTIStix2Utils
 
-urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
-
-
-class CustomJsonFormatter(jsonlogger.JsonFormatter):
-    def add_fields(self, log_record, record, message_dict):
-        super(CustomJsonFormatter, self).add_fields(log_record, record, message_dict)
-        if not log_record.get("timestamp"):
-            # This doesn't use record.created, so it is slightly off
-            now = datetime.datetime.utcnow().strftime("%Y-%m-%dT%H:%M:%S.%fZ")
-            log_record["timestamp"] = now
-        if log_record.get("level"):
-            log_record["level"] = log_record["level"].upper()
-        else:
-            log_record["level"] = record.levelname
-
 
 class File:
     def __init__(self, name, data, mime="text/plain"):
@@ -141,19 +123,8 @@ def __init__(
             raise ValueError("A TOKEN must be set")
 
         # Configure logger
-        log_level = log_level.upper()
-        LOGGER.setLevel(log_level)
-
-        if json_logging:
-            log_handler = logging.StreamHandler()
-            log_handler.setLevel(log_level)
-            formatter = CustomJsonFormatter(
-                "%(timestamp)s %(level)s %(name)s %(message)s"
-            )
-            log_handler.setFormatter(formatter)
-            logging.basicConfig(handlers=[log_handler], level=log_level, force=True)
-        else:
-            logging.basicConfig(level=log_level)
+        self.logger_class = logger(log_level.upper(), json_logging)
+        self.app_logger = self.logger_class("api")
 
         # Define API
         self.api_token = token
@@ -398,23 +369,6 @@ def fetch_opencti_file(self, fetch_uri, binary=False, serialize=False):
             return base64.b64encode(r.text).decode("utf-8")
         return r.text
 
-    def log(self, level, message):
-        """log a message with defined log level
-        :param level: must be a valid logging log level (debug, info, warning, error)
-        :type level: str
-        :param message: the message to log
-        :type message: str
-        """
-
-        if level == "debug":
-            LOGGER.debug(message)
-        elif level == "info":
-            LOGGER.info(message)
-        elif level == "warning":
-            LOGGER.warning(message)
-        elif level == "error":
-            LOGGER.error(message)
-
     def health_check(self):
         """submit an example request to the OpenCTI API.
 
@@ -426,7 +380,7 @@ def health_check(self):
             if test is not None:
                 return True
         except Exception as err:  # pylint: disable=broad-except
-            LOGGER.error("%s", err)
+            self.app_logger.error(str(err))
             return False
         return False
 
@@ -437,7 +391,7 @@ def get_logs_worker_config(self):
         rtype: dict
         """
 
-        LOGGER.info("Getting logs worker config...")
+        self.app_logger.info("Getting logs worker config...")
         query = """
             query LogsWorkerConfig {
                 logsWorkerConfig {
@@ -638,7 +592,7 @@ def upload_file(self, **kwargs):
         data = kwargs.get("data", None)
         mime_type = kwargs.get("mime_type", "text/plain")
         if file_name is not None:
-            LOGGER.info("Uploading a file.")
+            self.app_logger.info("Uploading a file.")
             query = """
                 mutation UploadImport($file: Upload!) {
                     uploadImport(file: $file) {
@@ -656,7 +610,7 @@ def upload_file(self, **kwargs):
 
             return self.query(query, {"file": (File(file_name, data, mime_type))})
         else:
-            LOGGER.error("[upload] Missing parameter: file_name")
+            self.app_logger.error("[upload] Missing parameter: file_name")
             return None
 
     def upload_pending_file(self, **kwargs):
@@ -673,7 +627,7 @@ def upload_pending_file(self, **kwargs):
         entity_id = kwargs.get("entity_id", None)
 
         if file_name is not None:
-            LOGGER.info("Uploading a file.")
+            self.app_logger.info("Uploading a file.")
             query = """
                     mutation UploadPending($file: Upload!, $entityId: String) {
                         uploadPending(file: $file, entityId: $entityId) {
@@ -693,7 +647,7 @@ def upload_pending_file(self, **kwargs):
                 {"file": (File(file_name, data, mime_type)), "entityId": entity_id},
             )
         else:
-            LOGGER.error("[upload] Missing parameter: file_name")
+            self.app_logger.error("[upload] Missing parameter: file_name")
             return None
 
     def get_stix_content(self, id):
@@ -703,7 +657,7 @@ def get_stix_content(self, id):
         rtype: dict
         """
 
-        LOGGER.info("Entity in JSON %s", id)
+        self.app_logger.info("Entity in JSON", {"id": id})
         query = """
             query StixQuery($id: String!) {
                 stix(id: $id)

pycti/api/opencti_api_connector.py

@@ -1,7 +1,6 @@
 import json
 from typing import Any, Dict
 
-from pycti.api import LOGGER
 from pycti.connector.opencti_connector import OpenCTIConnector
 
 
@@ -18,7 +17,7 @@ def list(self) -> Dict:
         :rtype: dict
         """
 
-        LOGGER.info("Getting connectors ...")
+        self.api.app_logger.info("Getting connectors ...")
         query = """
             query GetConnectors {
                 connectorsForWorker {

pycti/api/opencti_api_playbook.py

@@ -1,14 +1,11 @@
-from pycti.api import LOGGER
-
-
 class OpenCTIApiPlaybook:
     """OpenCTIApiPlaybook"""
 
     def __init__(self, api):
         self.api = api
 
     def playbook_step_execution(self, playbook: dict, bundle: str):
-        LOGGER.info("Executing playbook step %s", playbook["playbook_id"])
+        self.api.app_logger.info("Executing playbook step", playbook["playbook_id"])
         query = """
             mutation PlaybookStepExecution($execution_id: ID!, $execution_start: DateTime!, $data_instance_id: ID!, $playbook_id: ID!, $previous_step_id: ID!, $step_id: ID!, $previous_bundle: String!, $bundle: String!) {
                 playbookStepExecution(execution_id: $execution_id, execution_start: $execution_start, data_instance_id: $data_instance_id, playbook_id: $playbook_id, previous_step_id: $previous_step_id, step_id: $step_id, previous_bundle: $previous_bundle, bundle: $bundle)

pycti/api/opencti_api_work.py

@@ -1,8 +1,6 @@
 import time
 from typing import Dict, List
 
-from pycti.api import LOGGER
-
 
 class OpenCTIApiWork:
     """OpenCTIApiJob"""
@@ -11,7 +9,7 @@ def __init__(self, api):
         self.api = api
 
     def to_received(self, work_id: str, message: str):
-        LOGGER.info("Reporting work update_received %s", work_id)
+        self.api.app_logger.info("Reporting work update_received", {"work_id": work_id})
         query = """
             mutation workToReceived($id: ID!, $message: String) {
                 workEdit(id: $id) {
@@ -22,7 +20,9 @@ def to_received(self, work_id: str, message: str):
         self.api.query(query, {"id": work_id, "message": message})
 
     def to_processed(self, work_id: str, message: str, in_error: bool = False):
-        LOGGER.info("Reporting work update_processed %s", work_id)
+        self.api.app_logger.info(
+            "Reporting work update_processed", {"work_id": work_id}
+        )
         query = """
             mutation workToProcessed($id: ID!, $message: String, $inError: Boolean) {
                 workEdit(id: $id) {
@@ -33,7 +33,7 @@ def to_processed(self, work_id: str, message: str, in_error: bool = False):
         self.api.query(query, {"id": work_id, "message": message, "inError": in_error})
 
     def ping(self, work_id: str):
-        LOGGER.info("Ping work %s", work_id)
+        self.api.app_logger.info("Ping work", {"work_id": work_id})
         query = """
             mutation pingWork($id: ID!) {
                 workEdit(id: $id) {
@@ -44,7 +44,7 @@ def ping(self, work_id: str):
         self.api.query(query, {"id": work_id})
 
     def report_expectation(self, work_id: str, error):
-        LOGGER.info("Report expectation for %s", work_id)
+        self.api.app_logger.info("Report expectation", {"work_id": work_id})
         query = """
             mutation reportExpectation($id: ID!, $error: WorkErrorInput) {
                 workEdit(id: $id) {
@@ -55,10 +55,13 @@ def report_expectation(self, work_id: str, error):
         try:
             self.api.query(query, {"id": work_id, "error": error})
         except:
-            self.api.log("error", "Cannot report expectation")
+            self.api.app_logger.error("Cannot report expectation")
 
     def add_expectations(self, work_id: str, expectations: int):
-        LOGGER.info("Update action expectations %s - %s", work_id, expectations)
+        self.api.app_logger.info(
+            "Update action expectations",
+            {"work_id": work_id, "expectations": expectations},
+        )
         query = """
             mutation addExpectations($id: ID!, $expectations: Int) {
                 workEdit(id: $id) {
@@ -69,10 +72,10 @@ def add_expectations(self, work_id: str, expectations: int):
         try:
             self.api.query(query, {"id": work_id, "expectations": expectations})
         except:
-            self.api.log("error", "Cannot report expectation")
+            self.api.app_logger.error("Cannot report expectation")
 
     def initiate_work(self, connector_id: str, friendly_name: str) -> str:
-        LOGGER.info("Initiate work for %s", connector_id)
+        self.api.app_logger.info("Initiate work", {"connector_id": connector_id})
         query = """
             mutation workAdd($connectorId: String!, $friendlyName: String) {
                 workAdd(connectorId: $connectorId, friendlyName: $friendlyName) {
@@ -107,8 +110,8 @@ def wait_for_work_to_finish(self, work_id: str):
                 status = state["status"]
 
                 if state["errors"]:
-                    self.api.log(
-                        "error", f"Unexpected connector error {state['errors']}"
+                    self.api.app_logger.error(
+                        "Unexpected connector error", {"state_errors": state["errors"]}
                     )
                     return ""
 

pycti/connector/__init__.py

@@ -1,3 +0,0 @@
-import logging
-
-LOGGER = logging.getLogger(__name__)