o11y: Downgrade OTEL logs to DEBUG by default, make it configurable (#558)

The goal of this PR is primarily to downgrade OTEL logs to `DEBUG`, but
at the same time allow for keeping the previous behavior through the use
`OTEL_LOG_LEVEL` env var.

This should clean the logs that are currently flooded with time
statistics.

---------

Co-authored-by: Paweł Chmielak <[email protected]>

Author: Chmielok

CHANGELOG.md

@@ -1,3 +1,7 @@
+## 1.0.59
+
+* **o11y: Downgrade OTEL logs to `DEBUG` by default, make it configurable**
+
 ## 1.0.58
 
 * **o11y: Improved logging in connectors' operations with LoggingMixin class**

examples/local.py

@@ -14,7 +14,7 @@
 from unstructured_ingest.processes.embedder import EmbedderConfig
 from unstructured_ingest.processes.partitioner import PartitionerConfig
 
-base_path = Path(__file__).parent.parent.parent.parent
+base_path = Path(__file__).parent.parent
 docs_path = base_path / "example-docs"
 work_dir = base_path / "tmp_ingest" / CONNECTOR_TYPE
 output_path = work_dir / "output"

unstructured_ingest/__version__.py

@@ -1 +1 @@
-__version__ = "1.0.58"  # pragma: no cover
+__version__ = "1.0.59"  # pragma: no cover

unstructured_ingest/otel.py

@@ -1,3 +1,4 @@
+import logging
 import os
 from dataclasses import dataclass, field
 from typing import Callable, ClassVar, Optional, Protocol, Sequence
@@ -31,13 +32,27 @@ def export(self, spans: Sequence[ReadableSpan]) -> SpanExportResult:
             self.log_out(self.formatter(span))
         return SpanExportResult.SUCCESS
 
+def get_log_out() -> Callable:
+    level_names_mapping = {
+        'CRITICAL': logging.CRITICAL,
+        'FATAL': logging.FATAL,
+        'ERROR': logging.ERROR,
+        'WARN': logging.WARNING,
+        'WARNING': logging.WARNING,
+        'INFO': logging.INFO,
+        'DEBUG': logging.DEBUG,
+        'NOTSET': logging.NOTSET,
+    }
+    log_level = os.getenv("OTEL_LOG_LEVEL", "DEBUG").upper()
+    log_level_int = level_names_mapping.get(log_level, logging.DEBUG)
+    return lambda message: logger.log(log_level_int, message)
 
 @dataclass
 class OtelHandler:
     otel_endpoint: Optional[str] = None
     service_name: str = "unstructured-ingest"
     trace_provider: TracerProvider = field(init=False)
-    log_out: Callable = field(default=logger.info)
+    log_out: Callable = field(default=get_log_out())
     trace_context_key: ClassVar[str] = "_trace_context"
 
     def init_trace(self):