Merge pull request #296 from cmu-delphi/development

Changed logging format

Author: dmytrotsko

src/epiportal/logging_formatters.py

@@ -0,0 +1,65 @@
+"""
+Logging formatters for Elasticsearch-compatible structured output.
+"""
+
+import json
+import logging
+from datetime import datetime, timezone
+
+
+# Standard LogRecord attributes - excluded from extra when building JSON
+_RECORD_ATTRS = frozenset(
+    {
+        "name",
+        "msg",
+        "args",
+        "created",
+        "filename",
+        "funcName",
+        "levelname",
+        "levelno",
+        "lineno",
+        "module",
+        "msecs",
+        "pathname",
+        "process",
+        "processName",
+        "relativeCreated",
+        "stack_info",
+        "exc_info",
+        "exc_text",
+        "thread",
+        "threadName",
+        "message",
+        "asctime",
+        "taskName",
+    }
+)
+
+
+class JsonFormatter(logging.Formatter):
+    """
+    Format log records as single-line JSON for Elasticsearch.
+
+    Each log line is a valid JSON object with @timestamp, level, logger, message,
+    and any extra fields passed via logger.info(..., extra={...}).
+    """
+
+    def format(self, record: logging.LogRecord) -> str:
+        log_obj = {
+            "@timestamp": datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%S.%f")[:-3] + "Z",
+            "level": record.levelname,
+            "logger": record.name,
+            "message": record.getMessage(),
+        }
+
+        # Add extra fields (passed via extra= in log calls)
+        for key, value in record.__dict__.items():
+            if key not in _RECORD_ATTRS and value is not None:
+                log_obj[key] = value
+
+        # Add exception info if present
+        if record.exc_info:
+            log_obj["exception"] = self.formatException(record.exc_info)
+
+        return json.dumps(log_obj, default=str)

src/epiportal/middleware.py

@@ -2,7 +2,6 @@
 Request logging middleware that captures comprehensive data from all HTTP requests.
 """
 
-import json
 import logging
 import time
 import uuid
@@ -140,11 +139,11 @@ def process_response(self, request, response):
                 log_data["duration_ms"] = round(duration_ms, 2)
 
             logger.info(
-                "%s %s %s | %s",
+                "%s %s %s",
                 request.method,
                 request.path,
                 response.status_code,
-                json.dumps(log_data, default=str),
+                extra=log_data,
             )
 
         except Exception as e:

src/epiportal/settings.py

@@ -237,6 +237,9 @@
             'format': '[%(asctime)s] %(levelname)s | epiportal.requests | %(message)s',
             'datefmt': '%Y-%m-%d %H:%M:%S',
         },
+        'json': {
+            '()': 'epiportal.logging_formatters.JsonFormatter',
+        },
     },
     'handlers': {
         'console': {
@@ -246,7 +249,7 @@
         },
         'request_console': {
             'class': 'logging.StreamHandler',
-            'formatter': 'request_verbose',
+            'formatter': 'json',
             'stream': sys.stdout,
         },
     },
@@ -266,7 +269,8 @@
 
 if DEBUG:
     for logger in LOGGING['loggers']:
-        LOGGING['loggers'][logger]['handlers'] = ['console']
+        if logger != 'epiportal.requests':
+            LOGGING['loggers'][logger]['handlers'] = ['console']
 
 
 # DRF Spectacular settings