Add min_level argument to logfire.configure. (#1265)

Co-authored-by: David Montague <[email protected]>

Author: alexmojaki

docs/guides/onboarding-checklist/add-manual-tracing.md

@@ -250,14 +250,6 @@ The following methods exist for creating logs with different levels:
 - `logfire.error`
 - `logfire.fatal`
 
-You can set the minimum level used for console logging (`info` by default) with [`logfire.configure`][logfire.configure], e.g:
-
-```python
-import logfire
-
-logfire.configure(console=logfire.ConsoleOptions(min_log_level='debug'))
-```
-
 To log a message with a variable level you can use `logfire.log`, e.g. `logfire.log('info', 'This is an info log')` is equivalent to `logfire.info('This is an info log')`.
 
 Spans are level `info` by default. You can change this with the `_level` argument, e.g. `with logfire.span('This is a debug span', _level='debug'):`. You can also change the level after the span has started but before it's finished with [`span.set_level`][logfire.LogfireSpan.set_level], e.g:
@@ -270,3 +262,44 @@ with logfire.span('Doing a thing') as span:
 ```
 
 If a span finishes with an unhandled exception, then in addition to recording a traceback as described above, the span's log level will be set to `error`. This will not happen when using the [`span.record_exception`][logfire.LogfireSpan.record_exception] method.
+
+To skip creating logs/spans below a certain level, use the [`min_level`][logfire.configure(min_level)] argument to
+`logfire.configure`, e.g:
+
+```python
+import logfire
+
+logfire.configure(min_level='info')
+```
+
+For spans, this only applies when `_level` is explicitly specified in `logfire.span`.
+Setting the level after will be ignored by this.
+If a span is not created because of the minimum level, this has no effect on parents or children.
+For example, this code:
+
+```python
+import logfire
+
+logfire.configure(min_level='info')
+
+with logfire.span('root') as root:
+    root.set_level('debug')  # (1)!
+    with logfire.span('debug span excluded', _level='debug'):  # (2)!
+        logfire.info('info message')
+```
+
+1. This span has already been created with the default level of `info`, so `min_level` won't affect it.
+   It's also logged to the console because that happens at creation time.
+   But it will show in the Live view as having level `debug`.
+2. This span is not created because its level is below the minimum.
+   That makes this line a complete no-op.
+
+creates, logs, and sends the `root` span and the `info message` log, with the log being a direct child of the span.
+
+To set the minimum level for console logging only (`info` by default):
+
+```python
+import logfire
+
+logfire.configure(console=logfire.ConsoleOptions(min_log_level='debug'))
+```

logfire/_internal/config.py

@@ -69,6 +69,7 @@
 from .client import InvalidProjectName, LogfireClient, ProjectAlreadyExists
 from .config_params import ParamManager, PydanticPluginRecordValues
 from .constants import (
+    LEVEL_NUMBERS,
     RESOURCE_ATTRIBUTES_CODE_ROOT_PATH,
     RESOURCE_ATTRIBUTES_CODE_WORK_DIR,
     RESOURCE_ATTRIBUTES_DEPLOYMENT_ENVIRONMENT_NAME,
@@ -268,6 +269,7 @@ def configure(  # noqa: D417
     scrubbing: ScrubbingOptions | Literal[False] | None = None,
     inspect_arguments: bool | None = None,
     sampling: SamplingOptions | None = None,
+    min_level: int | LevelName | None = None,
     add_baggage_to_attributes: bool = True,
     code_source: CodeSource | None = None,
     distributed_tracing: bool | None = None,
@@ -320,6 +322,14 @@ def configure(  # noqa: D417
 
             Defaults to `True` if and only if the Python version is at least 3.11.
 
+        min_level:
+            Minimum log level for logs and spans to be created. By default, all logs and spans are created.
+            For example, set to 'info' to only create logs with level 'info' or higher, thus filtering out debug logs.
+            For spans, this only applies when `_level` is explicitly specified in `logfire.span`.
+            Changing the level of a span _after_ it is created will be ignored by this.
+            If a span is not created, this has no effect on the current active span, or on logs/spans created inside the
+            filtered `logfire.span` context manager.
+            If set to `None`, uses the `LOGFIRE_MIN_LEVEL` environment variable; if that is not set, there is no minimum level.
         sampling: Sampling options. See the [sampling guide](https://logfire.pydantic.dev/docs/guides/advanced/sampling/).
         add_baggage_to_attributes: Set to `False` to prevent OpenTelemetry Baggage from being added to spans as attributes.
             See the [Baggage documentation](https://logfire.pydantic.dev/docs/reference/advanced/baggage/) for more details.
@@ -456,6 +466,7 @@ def configure(  # noqa: D417
         additional_span_processors=additional_span_processors,
         scrubbing=scrubbing,
         inspect_arguments=inspect_arguments,
+        min_level=min_level,
         sampling=sampling,
         add_baggage_to_attributes=add_baggage_to_attributes,
         code_source=code_source,
@@ -514,6 +525,9 @@ class _LogfireConfigData:
     sampling: SamplingOptions
     """Sampling options."""
 
+    min_level: int
+    """Minimum log level for logs and spans to be created."""
+
     add_baggage_to_attributes: bool
     """Whether to add OpenTelemetry Baggage to span attributes."""
 
@@ -544,6 +558,7 @@ def _load_configuration(
         scrubbing: ScrubbingOptions | Literal[False] | None,
         inspect_arguments: bool | None,
         sampling: SamplingOptions | None,
+        min_level: int | LevelName | None,
         add_baggage_to_attributes: bool,
         code_source: CodeSource | None,
         distributed_tracing: bool | None,
@@ -561,6 +576,12 @@ def _load_configuration(
         self.inspect_arguments = param_manager.load_param('inspect_arguments', inspect_arguments)
         self.distributed_tracing = param_manager.load_param('distributed_tracing', distributed_tracing)
         self.ignore_no_config = param_manager.load_param('ignore_no_config')
+        min_level = param_manager.load_param('min_level', min_level)
+        if min_level is None:
+            min_level = 0
+        elif isinstance(min_level, str):
+            min_level = LEVEL_NUMBERS[min_level]
+        self.min_level = min_level
         self.add_baggage_to_attributes = add_baggage_to_attributes
 
         # We save `scrubbing` just so that it can be serialized and deserialized.
@@ -647,6 +668,7 @@ def __init__(
         scrubbing: ScrubbingOptions | Literal[False] | None = None,
         inspect_arguments: bool | None = None,
         sampling: SamplingOptions | None = None,
+        min_level: int | LevelName | None = None,
         add_baggage_to_attributes: bool = True,
         code_source: CodeSource | None = None,
         distributed_tracing: bool | None = None,
@@ -674,6 +696,7 @@ def __init__(
             scrubbing=scrubbing,
             inspect_arguments=inspect_arguments,
             sampling=sampling,
+            min_level=min_level,
             add_baggage_to_attributes=add_baggage_to_attributes,
             code_source=code_source,
             distributed_tracing=distributed_tracing,
@@ -712,6 +735,7 @@ def configure(
         scrubbing: ScrubbingOptions | Literal[False] | None,
         inspect_arguments: bool | None,
         sampling: SamplingOptions | None,
+        min_level: int | LevelName | None,
         add_baggage_to_attributes: bool,
         code_source: CodeSource | None,
         distributed_tracing: bool | None,
@@ -733,6 +757,7 @@ def configure(
                 scrubbing,
                 inspect_arguments,
                 sampling,
+                min_level,
                 add_baggage_to_attributes,
                 code_source,
                 distributed_tracing,
@@ -1053,6 +1078,7 @@ def fix_pid():  # pragma: no cover
                 self._logger_provider.shutdown()
 
             self._logger_provider.set_provider(logger_provider)
+            self._logger_provider.set_min_level(self.min_level)
 
             if self is GLOBAL_CONFIG and not self._has_set_providers:
                 self._has_set_providers = True

logfire/_internal/config_params.py

@@ -55,6 +55,8 @@ class _DefaultCallback:
 # fmt: off
 SEND_TO_LOGFIRE = ConfigParam(env_vars=['LOGFIRE_SEND_TO_LOGFIRE'], allow_file_config=True, default=_send_to_logfire_default, tp=Union[bool, Literal['if-token-present']])
 """Whether to send spans to Logfire."""
+MIN_LEVEL = ConfigParam(env_vars=['LOGFIRE_MIN_LEVEL'], allow_file_config=True, default=None, tp=LevelName)
+"""Minimum log level for logs and spans to be created. By default, all logs and spans are created."""
 TOKEN = ConfigParam(env_vars=['LOGFIRE_TOKEN'])
 """Token for the Logfire API."""
 SERVICE_NAME = ConfigParam(env_vars=['LOGFIRE_SERVICE_NAME', OTEL_SERVICE_NAME], allow_file_config=True, default='')
@@ -107,6 +109,7 @@ class _DefaultCallback:
 CONFIG_PARAMS = {
     'base_url': BASE_URL,
     'send_to_logfire': SEND_TO_LOGFIRE,
+    'min_level': MIN_LEVEL,
     'token': TOKEN,
     'service_name': SERVICE_NAME,
     'service_version': SERVICE_VERSION,

logfire/_internal/constants.py

@@ -4,7 +4,6 @@
 from typing import Literal
 
 from opentelemetry.context import create_key
-from opentelemetry.util import types as otel_types
 
 LOGFIRE_ATTRIBUTES_NAMESPACE = 'logfire'
 """Namespace within OTEL attributes used by logfire."""
@@ -90,7 +89,7 @@
 
 
 # This is in this file to encourage using it instead of setting these attributes manually.
-def log_level_attributes(level: LevelName | int) -> dict[str, otel_types.AttributeValue]:
+def log_level_attributes(level: LevelName | int) -> dict[str, int]:
     if isinstance(level, str):
         if level not in LEVEL_NUMBERS:
             warnings.warn(f'Invalid log level name: {level!r}')

logfire/_internal/integrations/fastapi.py

@@ -294,7 +294,6 @@ async def run_endpoint_function(
                 # both `http.method` and `http.route`.
                 **{'http.route': request.scope['route'].path},
                 **stack_info,
-                _level='debug',
             )
         else:
             extra_span = NoopSpan()

logfire/_internal/logs.py

@@ -8,7 +8,9 @@
 from weakref import WeakSet
 
 from opentelemetry import trace
-from opentelemetry._logs import Logger, LoggerProvider, LogRecord, NoOpLoggerProvider
+from opentelemetry._logs import Logger, LoggerProvider, LogRecord, NoOpLoggerProvider, SeverityNumber
+
+from logfire._internal.constants import LEVEL_NUMBERS
 
 if TYPE_CHECKING:
     from opentelemetry.util.types import _ExtendedAttributes  # type: ignore
@@ -23,6 +25,7 @@ class ProxyLoggerProvider(LoggerProvider):
     loggers: WeakSet[ProxyLogger] = dataclasses.field(default_factory=WeakSet)  # type: ignore[reportUnknownVariableType]
     lock: Lock = dataclasses.field(default_factory=Lock)
     suppressed_scopes: set[str] = dataclasses.field(default_factory=set)  # type: ignore[reportUnknownVariableType]
+    min_level: int = 0
 
     def get_logger(
         self,
@@ -37,10 +40,16 @@ def get_logger(
             else:
                 provider = self.provider
             inner_logger = provider.get_logger(name, version, schema_url, attributes)
-            logger = ProxyLogger(inner_logger, name, version, schema_url, attributes)
+            logger = ProxyLogger(inner_logger, self.min_level, name, version, schema_url, attributes)
             self.loggers.add(logger)
             return logger
 
+    def set_min_level(self, min_level: int) -> None:
+        with self.lock:
+            self.min_level = min_level
+            for logger in self.loggers:
+                logger.min_level = min_level
+
     def suppress_scopes(self, *scopes: str) -> None:
         with self.lock:
             self.suppressed_scopes.update(scopes)
@@ -78,12 +87,23 @@ def wrapper(*args: Any, **kwargs: Any):
 @dataclass(eq=False)
 class ProxyLogger(Logger):
     logger: Logger
+    min_level: int
     name: str
     version: str | None = None
     schema_url: str | None = None
     attributes: _ExtendedAttributes | None = None
 
     def emit(self, record: LogRecord) -> None:
+        if record.severity_number is not None:
+            if record.severity_number.value < self.min_level:
+                return
+        elif record.severity_text and (level_name := record.severity_text.lower()) in LEVEL_NUMBERS:
+            level_number = LEVEL_NUMBERS[level_name]
+            if level_number < self.min_level:
+                return
+            # record.severity_number is known to be None here, so we can safely set it
+            record.severity_number = SeverityNumber(level_number)
+
         if not record.trace_id:
             span_context = trace.get_current_span().get_span_context()
             record.trace_id = span_context.trace_id

logfire/_internal/main.py

@@ -179,6 +179,14 @@ def _span(
         _links: Sequence[tuple[SpanContext, otel_types.Attributes]] = (),
     ) -> LogfireSpan:
         try:
+            if _level is not None:
+                level_attributes = log_level_attributes(_level)
+                level_num = level_attributes[ATTRIBUTES_LOG_LEVEL_NUM_KEY]
+                if level_num < self.config.min_level:
+                    return NoopSpan()  # type: ignore
+            else:
+                level_attributes = None
+
             stack_info = get_user_stack_info()
             merged_attributes = {**stack_info, **attributes}
 
@@ -215,8 +223,8 @@ def _span(
             if sample_rate is not None and sample_rate != 1:  # pragma: no cover
                 otlp_attributes[ATTRIBUTES_SAMPLE_RATE_KEY] = sample_rate
 
-            if _level is not None:
-                otlp_attributes.update(log_level_attributes(_level))
+            if level_attributes is not None:
+                otlp_attributes.update(level_attributes)
 
             return LogfireSpan(
                 _span_name or msg_template,
@@ -673,6 +681,11 @@ def log(
             console_log: Whether to log to the console, defaults to `True`.
         """
         with handle_internal_errors:
+            level_attributes = log_level_attributes(level)
+            level_num = level_attributes[ATTRIBUTES_LOG_LEVEL_NUM_KEY]
+            if level_num < self.config.min_level:
+                return
+
             stack_info = get_user_stack_info()
 
             attributes = attributes or {}
@@ -706,7 +719,7 @@ def log(
             otlp_attributes = prepare_otlp_attributes(merged_attributes)
             otlp_attributes = {
                 ATTRIBUTES_SPAN_TYPE_KEY: 'log',
-                **log_level_attributes(level),
+                **level_attributes,
                 ATTRIBUTES_MESSAGE_TEMPLATE_KEY: msg_template,
                 ATTRIBUTES_MESSAGE_KEY: msg,
                 **otlp_attributes,

tests/otel_integrations/test_fastapi.py

@@ -450,7 +450,6 @@ def test_path_param(client: TestClient, exporter: TestExporter) -> None:
                     'logfire.span_type': 'pending_span',
                     'logfire.json_schema': '{"type":"object","properties":{"method":{},"http.route":{}}}',
                     'logfire.msg': 'GET /with_path_param/{param} (with_path_param)',
-                    'logfire.level_num': 5,
                     'logfire.pending_parent_id': '0000000000000001',
                 },
             },
@@ -470,7 +469,6 @@ def test_path_param(client: TestClient, exporter: TestExporter) -> None:
                     'logfire.span_type': 'span',
                     'logfire.json_schema': '{"type":"object","properties":{"method":{},"http.route":{}}}',
                     'logfire.msg': 'GET /with_path_param/{param} (with_path_param)',
-                    'logfire.level_num': 5,
                 },
             },
             {
@@ -687,7 +685,6 @@ def test_fastapi_instrumentation(client: TestClient, exporter: TestExporter) ->
                     'logfire.json_schema': {'type': 'object', 'properties': {'method': {}, 'http.route': {}}},
                     'logfire.span_type': 'pending_span',
                     'logfire.msg': 'GET / (homepage)',
-                    'logfire.level_num': 5,
                     'logfire.pending_parent_id': '0000000000000003',
                 },
             },
@@ -723,7 +720,6 @@ def test_fastapi_instrumentation(client: TestClient, exporter: TestExporter) ->
                     'logfire.span_type': 'span',
                     'logfire.json_schema': {'type': 'object', 'properties': {'method': {}, 'http.route': {}}},
                     'logfire.msg': 'GET / (homepage)',
-                    'logfire.level_num': 5,
                 },
             },
             {
@@ -1086,7 +1082,6 @@ def test_get_fastapi_arguments(client: TestClient, exporter: TestExporter) -> No
                     'logfire.msg_template': '{method} {http.route} ({code.function})',
                     'logfire.msg': 'GET /other (other_route)',
                     'logfire.json_schema': '{"type":"object","properties":{"method":{},"http.route":{}}}',
-                    'logfire.level_num': 5,
                     'logfire.span_type': 'span',
                 },
             },
@@ -1222,7 +1217,6 @@ def test_first_lvl_subapp_fastapi_arguments(client: TestClient, exporter: TestEx
                     'logfire.msg_template': '{method} {http.route} ({code.function})',
                     'logfire.msg': 'GET /other (other_route)',
                     'logfire.json_schema': '{"type":"object","properties":{"method":{},"http.route":{}}}',
-                    'logfire.level_num': 5,
                     'logfire.span_type': 'span',
                 },
             },
@@ -1358,7 +1352,6 @@ def test_second_lvl_subapp_fastapi_arguments(client: TestClient, exporter: TestE
                     'logfire.msg_template': '{method} {http.route} ({code.function})',
                     'logfire.msg': 'GET /other (other_route)',
                     'logfire.json_schema': '{"type":"object","properties":{"method":{},"http.route":{}}}',
-                    'logfire.level_num': 5,
                     'logfire.span_type': 'span',
                 },
             },
@@ -1785,7 +1778,6 @@ def test_scrubbing(client: TestClient, exporter: TestExporter) -> None:
                     'logfire.msg_template': '{method} {http.route} ({code.function})',
                     'logfire.msg': 'GET /secret/{path_param} (get_secret)',
                     'logfire.json_schema': '{"type":"object","properties":{"method":{},"http.route":{}}}',
-                    'logfire.level_num': 5,
                     'logfire.span_type': 'span',
                 },
             },