Capture logs (#485)

* start custom logger

* initial logging support

* test coverage

* Validator refactor (#478)

* refactor validators to multiple files

* complete validator split

* fix one-line validator docstring (#484)

* docstring fix

* lint

---------

Co-authored-by: Nefertiti  Rogers <[email protected]>

* PR comments

* lint fixes

---------

Co-authored-by: zsimjee <[email protected]>
Co-authored-by: Nefertiti Rogers <[email protected]>
Co-authored-by: Nefertiti  Rogers <[email protected]>

Author: 4 people

.gitignore

@@ -20,6 +20,7 @@ dist/*
 .cache
 scratch/
 .coverage*
+coverage.xml
 test.db
 test.index
 htmlcov

Makefile

@@ -47,6 +47,9 @@ test-cov:
 view-test-cov:
 	poetry run pytest tests/ --cov=./guardrails/ --cov-report html && open htmlcov/index.html
 
+view-test-cov-file:
+	poetry run pytest tests/unit_tests/test_logger.py --cov=./guardrails/ --cov-report html && open htmlcov/index.html
+
 docs-serve:
 	poetry run mkdocs serve -a $(MKDOCS_SERVE_ADDR)
 

guardrails/classes/history/call.py

@@ -134,10 +134,13 @@ def reask_instructions(self) -> Stack[str]:
     #     # To allow chaining without getting AttributeErrors
     #     return Outputs()
 
-    # TODO
-    # @property
-    # def logs(self) -> Stack[str]:
-    #     """Returns all logs from all iterations as a stack"""
+    @property
+    def logs(self) -> Stack[str]:
+        """Returns all logs from all iterations as a stack."""
+        all_logs = []
+        for i in self.iterations:
+            all_logs.extend(i.logs)
+        return Stack(*all_logs)
 
     @property
     def tokens_consumed(self) -> Optional[int]:

guardrails/classes/history/iteration.py

@@ -6,12 +6,12 @@
 from rich.pretty import pretty_repr
 from rich.table import Table
 
+from guardrails.classes.generic.stack import Stack
 from guardrails.classes.history.inputs import Inputs
 from guardrails.classes.history.outputs import Outputs
+from guardrails.logger import get_scope_handler
 from guardrails.prompt.prompt import Prompt
 from guardrails.utils.logs_utils import ValidatorLogs
-
-# from guardrails.classes.stack import Stack
 from guardrails.utils.pydantic_utils import ArbitraryModel
 from guardrails.utils.reask_utils import ReAsk
 
@@ -27,10 +27,13 @@ class Iteration(ArbitraryModel):
         description="The outputs from the iteration/step.", default_factory=Outputs
     )
 
-    # TODO
-    # @property
-    # def logs() -> Stack[str]:
-    #     """Returns the logs from this iteration as a stack"""
+    @property
+    def logs(self) -> Stack[str]:
+        """Returns the logs from this iteration as a stack."""
+        scope = str(id(self))
+        scope_handler = get_scope_handler()
+        scoped_logs = scope_handler.get_logs(scope)
+        return Stack(*[log.getMessage() for log in scoped_logs])
 
     @property
     def tokens_consumed(self) -> Optional[int]:

guardrails/classes/history/outputs.py

@@ -19,7 +19,7 @@ class Outputs(ArbitraryModel):
         "as it was passed into validation.",
         default=None,
     )
-    validation_output: Optional[Union[str, Dict, ReAsk]] = Field(
+    validation_output: Optional[Union[str, ReAsk, Dict]] = Field(
         description="The output from the validation process.", default=None
     )
     validated_output: Optional[Union[str, Dict]] = Field(
@@ -48,6 +48,7 @@ def _all_empty(self) -> bool:
         return (
             self.llm_response_info is None
             and self.parsed_output is None
+            and self.validation_output is None
             and self.validated_output is None
             and len(self.reasks) == 0
             and len(self.validator_logs) == 0
@@ -79,6 +80,7 @@ def status(self) -> str:
         all_reasks_have_fixes = all(
             list(fail.fix_value is not None for fail in all_fail_results)
         )
+
         if self._all_empty() is True:
             return not_run_status
         elif self.error:

guardrails/datatypes.py

@@ -1,5 +1,4 @@
 import datetime
-import logging
 import warnings
 from dataclasses import dataclass
 from types import SimpleNamespace
@@ -16,9 +15,6 @@
 from guardrails.validator_base import Validator, ValidatorSpec
 from guardrails.validatorsattr import ValidatorsAttr
 
-logger = logging.getLogger(__name__)
-
-
 # TODO - deprecate these altogether
 deprecated_string_types = {"sql", "email", "url", "pythoncode"}
 

guardrails/guard.py

@@ -1,6 +1,5 @@
 import asyncio
 import contextvars
-import logging
 from typing import (
     Any,
     Awaitable,
@@ -24,15 +23,14 @@
 from guardrails.classes.history import Call
 from guardrails.classes.history.call_inputs import CallInputs
 from guardrails.llm_providers import get_async_llm_ask, get_llm_ask
+from guardrails.logger import logger, set_scope
 from guardrails.prompt import Instructions, Prompt
 from guardrails.rail import Rail
 from guardrails.run import AsyncRunner, Runner
 from guardrails.schema import Schema
 from guardrails.validators import Validator
 
-logger = logging.getLogger(__name__)
-actions_logger = logging.getLogger(f"{__name__}.actions")
-add_destinations(actions_logger.debug)
+add_destinations(logger.debug)
 
 
 class Guard(Generic[OT]):
@@ -61,6 +59,7 @@ def __init__(
         """Initialize the Guard."""
         self.rail = rail
         self.num_reasks = num_reasks
+        # TODO: Support a sink for history so that it is not solely held in memory
         self.history: Stack[Call] = Stack()
         self.base_model = base_model
 
@@ -315,6 +314,7 @@ def __call__(
             kwargs=kwargs,
         )
         call_log = Call(inputs=call_inputs)
+        set_scope(str(id(call_log)))
         self.history.push(call_log)
 
         # If the LLM API is async, return a coroutine
@@ -553,6 +553,7 @@ def parse(
             kwargs=kwargs,
         )
         call_log = Call(inputs=call_inputs)
+        set_scope(str(id(call_log)))
         self.history.push(call_log)
 
         # If the LLM API is async, return a coroutine

guardrails/logger.py

@@ -0,0 +1,109 @@
+import logging
+import logging.config
+from logging import Handler, LogRecord
+from typing import Dict, List, Optional
+
+# from src.modules.otel_logger import handler as otel_handler
+
+name = "guardrails-ai"
+base_scope = "base"
+all_scopes = "all"
+
+
+class ScopeHandler(Handler):
+    scope: str
+    scoped_logs: Dict[str, List[LogRecord]]
+
+    def __init__(self, level=logging.NOTSET, scope=base_scope):
+        super().__init__(level)
+        self.scope = scope
+        self.scoped_logs = {}
+
+    def emit(self, record: LogRecord) -> None:
+        logs = self.scoped_logs.get(self.scope, [])
+        logs.append(record)
+        self.scoped_logs[self.scope] = logs
+
+    def set_scope(self, scope: str = base_scope):
+        self.scope = scope
+
+    def get_all_logs(self) -> List[LogRecord]:
+        all_logs = []
+        for key in self.scoped_logs:
+            logs = self.scoped_logs.get(key, [])
+            all_logs.extend(logs)
+        return all_logs
+
+    def get_logs(self, scope: Optional[str] = None) -> List[LogRecord]:
+        scope = scope or self.scope
+        if scope == all_scopes:
+            return self.get_all_logs()
+        logs = self.scoped_logs.get(scope, [])
+        return logs
+
+
+class LoggerConfig:
+    def __init__(self, config={}, level=logging.NOTSET, scope=base_scope):
+        self.config = config
+        self.level = level
+        self.scope = scope
+
+
+_logger = logging.getLogger(name)
+handler = ScopeHandler()
+scoped_logs: Dict[str, List[LogRecord]] = {}
+logger_config = LoggerConfig()
+
+
+def get_logger():
+    _setup_handler(logger_config.level, logger_config.scope)
+
+    if logger_config.config:
+        logging.config.dictConfig(logger_config.config)
+
+    _logger.setLevel(logger_config.level)
+
+    get_scope_handler()
+
+    return _logger
+
+
+def set_config(config=None):
+    if config is not None:
+        logger_config.config = config
+        logging.config.dictConfig(logger_config.config)
+
+
+def set_level(level=None):
+    if level is not None:
+        logger_config.level = level
+        _logger.setLevel(level)
+
+
+def set_scope(scope: str = base_scope):
+    logger_config.scope = scope
+    scope_handler = get_scope_handler()
+    scope_handler.set_scope(scope)
+
+
+def _setup_handler(log_level=logging.NOTSET, scope=base_scope) -> ScopeHandler:
+    global handler
+    if not handler:
+        handler = ScopeHandler(log_level, scope)
+    return handler
+
+
+def get_scope_handler() -> ScopeHandler:
+    global _logger
+    try:
+        scope_handler: ScopeHandler = [
+            h for h in _logger.handlers if isinstance(h, ScopeHandler)
+        ][0]
+        return scope_handler
+    except IndexError:
+        hdlr = _setup_handler()
+        _logger.addHandler(hdlr)
+        return hdlr
+
+
+logger = get_logger()

guardrails/logging_utils.py

@@ -1,11 +1,7 @@
-import logging
-import logging.config
+from guardrails.logger import set_config, set_level
 
 
+# TODO: Support a sink for logs so that they are not solely held in memory
 def configure_logging(logging_config=None, log_level=None):
-    if logging_config is not None:
-        logging.config.dictConfig(logging_config)
-
-    if log_level is not None:
-        logger = logging.getLogger(__name__)
-        logger.setLevel(log_level)
+    set_config(logging_config)
+    set_level(log_level)

guardrails/run.py

@@ -1,5 +1,4 @@
 import copy
-import logging
 from typing import Any, Dict, List, Optional, Sequence, Tuple, Type, Union
 
 from eliot import add_destinations, start_action
@@ -8,14 +7,13 @@
 from guardrails.classes.history import Call, Inputs, Iteration, Outputs
 from guardrails.datatypes import verify_metadata_requirements
 from guardrails.llm_providers import AsyncPromptCallableBase, PromptCallableBase
+from guardrails.logger import logger, set_scope
 from guardrails.prompt import Instructions, Prompt
 from guardrails.schema import Schema
 from guardrails.utils.llm_response import LLMResponse
 from guardrails.utils.reask_utils import NonParseableReAsk, ReAsk, reasks_to_dict
 
-logger = logging.getLogger(__name__)
-actions_logger = logging.getLogger(f"{__name__}.actions")
-add_destinations(actions_logger.debug)
+add_destinations(logger.debug)
 
 
 class Runner:
@@ -200,6 +198,7 @@ def step(
         )
         outputs = Outputs()
         iteration = Iteration(inputs=inputs, outputs=outputs)
+        set_scope(str(id(iteration)))
         call_log.iterations.push(iteration)
 
         try: