feat: implement logging functionality in logger_task for API activity

- Added a new logger_task module to handle logging of API activity to MongoDB and PostgreSQL.
- Introduced functions for safely decoding request/response bodies and processing logs based on MongoDB availability.
- Refactored APITokenLogMiddleware to utilize the new logging functions, improving code organization and maintainability.

Author: pablohashescobar

apps/api/plane/bgtasks/logger_task.py

@@ -0,0 +1,93 @@
+# Python imports
+import logging
+from typing import Optional, Dict, Any
+
+# Third party imports
+from pymongo.collection import Collection
+
+# Django imports
+from plane.settings.mongo import MongoConnection
+from plane.utils.exception_logger import log_exception
+from plane.db.models import APIActivityLog
+
+
+logger = logging.getLogger("plane.worker")
+
+
+def get_mongo_collection() -> Optional[Collection]:
+    """
+    Returns the MongoDB collection for external API activity logs.
+    """
+    if not MongoConnection.is_configured():
+        logger.info("MongoDB not configured")
+        return None
+
+    try:
+        return MongoConnection.get_collection("api_activity_logs")
+    except Exception as e:
+        logger.error(f"Error getting MongoDB collection: {str(e)}")
+        log_exception(e)
+        return None
+
+
+def safe_decode_body(content: bytes) -> Optional[str]:
+    """
+    Safely decodes request/response body content, handling binary data.
+    Returns "[Binary Content]" if the content is binary, or a string representation of the content.
+    Returns None if the content is None or empty.
+    """
+    # If the content is None, return None
+    if content is None:
+        return None
+
+    # If the content is an empty bytes object, return None
+    if content == b"":
+        return None
+
+    # Check if content is binary by looking for common binary file signatures
+    if content.startswith(b"\x89PNG") or content.startswith(b"\xff\xd8\xff") or content.startswith(b"%PDF"):
+        return "[Binary Content]"
+
+    try:
+        return content.decode("utf-8")
+    except UnicodeDecodeError:
+        return "[Could not decode content]"
+
+
+def log_to_mongo(mongo_collection: Optional[Collection], log_document: Dict[str, Any]) -> bool:
+    """
+    Logs the request to MongoDB if available.
+    """
+
+    if mongo_collection is None:
+        return False
+
+    try:
+        mongo_collection.insert_one(log_document)
+        return True
+    except Exception as e:
+        log_exception(e)
+        return False
+
+
+def log_to_postgres(log_data: Dict[str, Any]) -> bool:
+    """
+    Fallback to logging to PostgreSQL if MongoDB is unavailable.
+    """
+    try:
+        APIActivityLog.objects.create(**log_data)
+        return True
+    except Exception as e:
+        log_exception(e)
+        return False
+
+
+def process_logs(log_data: Dict[str, Any], mongo_log: Dict[str, Any]):
+    """
+    Process logs to save to MongoDB or Postgres based on the configuration
+    """
+    mongo_collection = get_mongo_collection()
+    if mongo_collection is not None:
+        log_to_mongo(mongo_collection, mongo_log)
+    else:
+        log_to_postgres(log_data)

apps/api/plane/middleware/logger.py

@@ -14,6 +14,7 @@
 from plane.db.models import APIActivityLog
 from plane.settings.mongo import MongoConnection
 from plane.utils.exception_logger import log_exception
+from plane.bgtasks.logger_task import process_logs
 
 api_logger = logging.getLogger("plane.api.request")
 
@@ -79,30 +80,6 @@ class APITokenLogMiddleware:
 
     def __init__(self, get_response):
         self.get_response = get_response
-        self.mongo_available = False
-
-        # Initialize MongoDB collection
-        try:
-            self.mongo_collection = self.get_mongo_collection()
-            self.mongo_available = True if self.mongo_collection is not None else False
-        except Exception as e:
-            api_logger.error(f"Error getting MongoDB collection: {str(e)}")
-            log_exception(e)
-
-    def get_mongo_collection(self):
-        """
-        Returns the MongoDB collection for API activity logs.
-        """
-        if not MongoConnection.is_configured():
-            api_logger.info("MongoDB not configured")
-            return None
-
-        try:
-            return MongoConnection.get_collection("api_activity_logs")
-        except Exception as e:
-            api_logger.error(f"Error getting MongoDB collection: {str(e)}")
-            log_exception(e)
-            return None
 
     def __call__(self, request):
         request_body = request.body
@@ -132,34 +109,6 @@ def _safe_decode_body(self, content):
         except UnicodeDecodeError:
             return "[Could not decode content]"
 
-    def log_to_mongo(self, log_document):
-        """
-        Logs the request to MongoDB if available.
-        """
-
-        if not self.mongo_available or self.mongo_collection is None:
-            return False
-
-        try:
-            self.mongo_collection.insert_one(log_document)
-            return True
-        except Exception as e:
-            log_exception(e)
-            self.mongo_collection = self.get_mongo_collection()
-            self.mongo_available = True if self.mongo_collection is not None else False
-            return False
-
-    def log_to_postgres(self, log_data):
-        """
-        Fallback to logging to PostgreSQL if MongoDB is unavailable.
-        """
-        try:
-            APIActivityLog.objects.create(**log_data)
-            return True
-        except Exception as e:
-            log_exception(e)
-            return False
-
     def process_request(self, request, response, request_body):
         api_key_header = "X-Api-Key"
         api_key = request.headers.get(api_key_header)
@@ -195,10 +144,7 @@ def process_request(self, request, response, request_body):
                 "updated_by": user_id,
             }
 
-            # Log to MongoDB if available
-            if not self.log_to_mongo(mongo_log):
-                # Fallback to logging to PostgreSQL
-                self.log_to_postgres(log_data)
+            process_logs.delay(log_data=log_data, mongo_log=mongo_log)
 
         except Exception as e:
             log_exception(e)