Refactor/logging (#98)

* allow setting random seed for a pipeline from python api

* remove CLI

* fix typing & default calue of OptimizationConfig

* upd tests (remove outdated `force_multilabel` argument)

* remove CLI tutorials

* refactor configs to pydantic

* fix typing

* finally configure post initialization

* upd docstrings

* remove CLI from docs

* remove hydra from poetry

* remove omegaconf

* add logging config, setup and formatter

* fix custom log level issue

* add logging to file by demand

* fix typing

* remove unnecessary formatter

* add docs for `setup_logging`

* add tutorial on logging

* minor bugfix

* minor bugfix

* another attempt

* fix codestyle

Author: voorhs

autointent/__init__.py

@@ -1,5 +1,6 @@
 """This is AutoIntent API reference."""
 
+from ._logging import setup_logging
 from ._ranker import Ranker
 from ._embedder import Embedder
 from ._vector_index import VectorIndex
@@ -8,5 +9,4 @@
 from .context import Context
 from ._pipeline import Pipeline
 
-
-__all__ = ["Context", "Dataset", "Embedder", "Hasher", "Pipeline", "Ranker", "VectorIndex"]
+__all__ = ["Context", "Dataset", "Embedder", "Hasher", "Pipeline", "Ranker", "VectorIndex", "setup_logging"]

autointent/_logging/__init__.py

@@ -0,0 +1,3 @@
+from .setup import setup_logging
+
+__all__ = ["setup_logging"]

autointent/_logging/config.yaml

@@ -0,0 +1,27 @@
+version: 1
+disable_existing_loggers: false
+formatters:
+  simple:
+    format: '%(levelname)s: %(message)s'
+    datefmt: '%Y-%m-%dT%H:%M:%S%z'
+  json:
+    (): autointent._logging.formatter.JSONFormatter
+    fmt_keys:
+      level: levelname
+      message: message
+      timestamp: timestamp
+      logger: name
+      module: module
+      function: funcName
+      line: lineno
+      thread_name: threadName
+handlers:
+  stdout:
+    class: logging.StreamHandler
+    formatter: simple
+    stream: ext://sys.stdout
+loggers:
+  root:
+    level: DEBUG
+    handlers:
+    - stdout

autointent/_logging/formatter.py

@@ -0,0 +1,68 @@
+import datetime as dt
+import json
+import logging
+from typing import Any
+
+LOG_RECORD_BUILTIN_ATTRS = {
+    "args",
+    "asctime",
+    "created",
+    "exc_info",
+    "exc_text",
+    "filename",
+    "funcName",
+    "levelname",
+    "levelno",
+    "lineno",
+    "module",
+    "msecs",
+    "message",
+    "msg",
+    "name",
+    "pathname",
+    "process",
+    "processName",
+    "relativeCreated",
+    "stack_info",
+    "thread",
+    "threadName",
+    "taskName",
+}
+
+
+class JSONFormatter(logging.Formatter):
+    """This is a custom formatter for saving logging records as a json."""
+
+    def __init__(
+        self,
+        *,
+        fmt_keys: dict[str, str] | None = None,
+    ) -> None:
+        super().__init__()
+        self.fmt_keys = fmt_keys if fmt_keys is not None else {}
+
+    def format(self, record: logging.LogRecord) -> str:
+        message = self._prepare_log_dict(record)
+        return json.dumps(message, default=str)
+
+    def _prepare_log_dict(self, record: logging.LogRecord) -> dict[str, Any]:
+        always_fields = {
+            "message": record.getMessage(),
+            "timestamp": dt.datetime.fromtimestamp(record.created, tz=dt.timezone.utc).isoformat(),
+        }
+        if record.exc_info is not None:
+            always_fields["exc_info"] = self.formatException(record.exc_info)
+
+        if record.stack_info is not None:
+            always_fields["stack_info"] = self.formatStack(record.stack_info)
+
+        message = {
+            key: msg_val if (msg_val := always_fields.pop(val, None)) is not None else getattr(record, val)
+            for key, val in self.fmt_keys.items()
+        }
+        message.update(always_fields)
+
+        extra_fields = {key: val for key, val in record.__dict__.items() if key not in LOG_RECORD_BUILTIN_ATTRS}
+        message.update(extra_fields)
+
+        return message

autointent/_logging/setup.py

@@ -0,0 +1,41 @@
+import importlib.resources as ires
+import logging.config
+import logging.handlers
+from pathlib import Path
+
+import yaml
+
+from autointent.custom_types import LogLevel
+
+
+def setup_logging(level: LogLevel | str, log_filename: Path | str | None = None) -> None:
+    """
+    Set stdout and file handlers for logging autointent internal actions.
+
+    The first parameter affects the logs to the standard output stream. The second parameter is optional.
+    If it is specified, then the "DEBUG" messages are logged to the file,
+    regardless of what is specified by the first parameter.
+
+    :param level: one of "DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"
+    :param log_to_filepath: specify location of logfile, omit extension as suffix ``.log.jsonl`` will be appended.
+    """
+    config_file = ires.files("autointent._logging").joinpath("config.yaml")
+    with config_file.open() as f_in:
+        config = yaml.safe_load(f_in)
+
+    level = LogLevel(level)
+    config["handlers"]["stdout"]["level"] = level.value
+
+    if log_filename is not None:
+        config["loggers"]["root"]["handlers"].append("file")
+
+        filename = str(log_filename) + ".log.jsonl"
+        config["handlers"]["file"] = {
+            "class": "logging.FileHandler",
+            "level": "DEBUG",
+            "formatter": "json",
+            "filename": filename,
+        }
+        Path(filename).parent.mkdir(parents=True, exist_ok=True)
+
+    logging.config.dictConfig(config)

autointent/context/optimization_info/_logger.py

@@ -4,6 +4,7 @@
 trials, and modules during the pipeline's execution.
 """
 
+import logging
 from dataclasses import dataclass, field
 from typing import TYPE_CHECKING, Any, Literal
 
@@ -14,7 +15,6 @@
 from autointent.custom_types import NodeType
 
 from ._data_models import Artifact, Artifacts, RetrieverArtifact, ScorerArtifact, Trial, Trials, TrialsIds
-from ._logger import get_logger
 
 if TYPE_CHECKING:
     from autointent.modules.abc import Module
@@ -59,7 +59,7 @@ class OptimizationInfo:
 
     def __init__(self) -> None:
         """Initialize optimization info."""
-        self._logger = get_logger()
+        self._logger = logging.getLogger(__name__)
 
         self.artifacts = Artifacts()
         self.trials = Trials()
@@ -98,7 +98,7 @@ def log_module_optimization(
             module_dump_dir=module_dump_dir,
         )
         self.trials.add_trial(node_type, trial)
-        self._logger.info(trial.model_dump())
+        self._logger.debug("module %s fitted and saved to optimization info", module_name, extra=trial.model_dump())
 
         if module:
             self.modules.add_module(node_type, module)

autointent/context/optimization_info/_optimization_info.py

@@ -0,0 +1,76 @@
+# %% [markdown]
+"""
+# Logging to stdout and file
+
+This guide will teach you how to configure logging in AutoIntent. By default, it is fully disabled.
+
+It will be demonstrated on toy search_space example:
+"""
+
+# %%
+from pathlib import Path
+
+from autointent import Dataset, Pipeline
+from autointent.configs import LoggingConfig
+
+search_space = [
+    {
+        "node_type": "scoring",
+        "metric": "scoring_roc_auc",
+        "search_space": [
+            {
+                "module_name": "knn",
+                "k": [1],
+                "weights": ["uniform"],
+                "embedder_name": ["avsolatorio/GIST-small-Embedding-v0"],
+            },
+        ],
+    },
+    {
+        "node_type": "decision",
+        "metric": "decision_accuracy",
+        "search_space": [
+            {"module_name": "threshold", "thresh": [0.5]},
+            {"module_name": "argmax"},
+        ],
+    },
+]
+
+log_config = LoggingConfig(dirpath=Path("logging_tutorial"))
+pipeline_optimizer = Pipeline.from_search_space(search_space)
+pipeline_optimizer.set_config(log_config)
+
+dataset = Dataset.from_hub("AutoIntent/clinc150_subset")
+
+# %% [markdown]
+"""
+## Fully Custom Logging
+
+One can fully customize logging via python's standard module [`logging`](https://docs.python.org/3/library/logging.html). Everything you need to do is configure it before AutoIntent execution:
+"""
+# %%
+import logging
+
+logging.basicConfig(level="INFO")
+pipeline_optimizer.fit(dataset)
+
+# %% [markdown]
+"""
+See external tutorials and guides about `logging` module.
+"""
+
+# %% [markdown]
+"""
+## Export from AutoIntent
+
+If you don't have to customize logging, you can export our configuration. Everything you need to do is setup it before AutoIntent execution:
+"""
+
+# %%
+from autointent import setup_logging
+
+setup_logging("INFO", log_filename="tests/logs/my_exp")
+# %%
+"""
+The first parameter affects the logs to the standard output stream. The second parameter is optional. If it is specified, then the "DEBUG" messages are logged to the file, regardless of what is specified by the first parameter.
+"""