add logging config, setup and formatter

Author: voorhs

autointent/__init__.py

@@ -1,5 +1,6 @@
 """This is AutoIntent API reference."""
 
+from ._logging import setup_logging
 from ._ranker import Ranker
 from ._embedder import Embedder
 from ._vector_index import VectorIndex
@@ -8,5 +9,4 @@
 from .context import Context
 from ._pipeline import Pipeline
 
-
-__all__ = ["Context", "Dataset", "Embedder", "Hasher", "Pipeline", "Ranker", "VectorIndex"]
+__all__ = ["Context", "Dataset", "Embedder", "Hasher", "Pipeline", "Ranker", "VectorIndex", "setup_logging"]

autointent/_logging/__init__.py

@@ -0,0 +1,3 @@
+from .setup import setup_logging
+
+__all__ = ["setup_logging"]

autointent/_logging/config.yaml

@@ -0,0 +1,29 @@
+version: 1
+disable_existing_loggers: false
+formatters:
+  simple:
+    format: '%(levelname)s: %(message)s'
+    datefmt: '%Y-%m-%dT%H:%M:%S%z'
+  json:
+    (): autointent._logging.formatter.JSONFormatter
+    fmt_keys:
+      level: levelname
+      message: message
+      timestamp: timestamp
+      logger: name
+      module: module
+      function: funcName
+      line: lineno
+      thread_name: threadName
+handlers:
+  stderr:
+    class: logging.StreamHandler
+    level: INFO
+    formatter: json
+    stream: ext://sys.stdout
+loggers:
+  root:
+    level: DEBUG
+    handlers:
+    - stderr
+    - file

autointent/_logging/formatter.py

@@ -0,0 +1,69 @@
+import datetime as dt
+import json
+import logging
+from typing import Any, override
+
+LOG_RECORD_BUILTIN_ATTRS = {
+    "args",
+    "asctime",
+    "created",
+    "exc_info",
+    "exc_text",
+    "filename",
+    "funcName",
+    "levelname",
+    "levelno",
+    "lineno",
+    "module",
+    "msecs",
+    "message",
+    "msg",
+    "name",
+    "pathname",
+    "process",
+    "processName",
+    "relativeCreated",
+    "stack_info",
+    "thread",
+    "threadName",
+    "taskName",
+}
+
+
+class JSONFormatter(logging.Formatter):
+    """This is a custom formatter for saving logging records as a json."""
+
+    def __init__(
+        self,
+        *,
+        fmt_keys: dict[str, str] | None = None,
+    ) -> None:
+        super().__init__()
+        self.fmt_keys = fmt_keys if fmt_keys is not None else {}
+
+    @override
+    def format(self, record: logging.LogRecord) -> str:
+        message = self._prepare_log_dict(record)
+        return json.dumps(message, default=str)
+
+    def _prepare_log_dict(self, record: logging.LogRecord) -> dict[str, Any]:
+        always_fields = {
+            "message": record.getMessage(),
+            "timestamp": dt.datetime.fromtimestamp(record.created, tz=dt.timezone.utc).isoformat(),
+        }
+        if record.exc_info is not None:
+            always_fields["exc_info"] = self.formatException(record.exc_info)
+
+        if record.stack_info is not None:
+            always_fields["stack_info"] = self.formatStack(record.stack_info)
+
+        message = {
+            key: msg_val if (msg_val := always_fields.pop(val, None)) is not None else getattr(record, val)
+            for key, val in self.fmt_keys.items()
+        }
+        message.update(always_fields)
+
+        extra_fields = {key: val for key, val in record.__dict__.items() if key not in LOG_RECORD_BUILTIN_ATTRS}
+        message.update(extra_fields)
+
+        return message

autointent/_logging/setup.py

@@ -0,0 +1,15 @@
+import importlib.resources as ires
+import json
+import logging.config
+import logging.handlers
+
+from autointent.custom_types import LogLevel
+
+
+def setup_logging(level: LogLevel) -> None:
+    config_file = ires.files("autointent._logging").joinpath("config.json")
+    with config_file.open() as f_in:
+        config = json.load(f_in)
+
+    logging.config.dictConfig(config)
+    logging.basicConfig(level=level.value)