o11y: Improve connectors logging (#557)

# Enhance logging in connectors
This PR is enhancing the logging in connectors by introducing the
LoggingMixin class that handles logging for standard connectors'
operations sanitizes restricted fields like filenames (optional)

---------

Co-authored-by: Copilot <[email protected]>
Co-authored-by: Paweł Chmielak <[email protected]>
Co-authored-by: Paweł Chmielak <[email protected]>

Author: 4 people

CHANGELOG.md

@@ -1,16 +1,19 @@
+## 1.0.58
+
+* **o11y: Improved logging in connectors' operations with LoggingMixin class**
+
 ## 1.0.57
 
-- **test: Longer interval for pinecone integration tests**
+* **test: Longer interval for pinecone integration tests**
 
 ## 1.0.56
 
-- **Fix: set correct display_name in HtmlMixin produced FileData**
+* **Fix: set correct display_name in HtmlMixin produced FileData**
 
 ## 1.0.55
 
 * **Fix: add precheck method to SharePoint connector**
 
-
 ## 1.0.54
 
 * **Fix bump Togetherai dependency**

test/integration/connectors/test_sharepoint.py

@@ -333,62 +333,66 @@ def base_sharepoint_config():
     """Base SharePoint config for testing."""
     config = sharepoint_config()
     return {
-        'client_id': config.client_id,
-        'tenant': config.tenant,
-        'site': "https://unstructuredio.sharepoint.com/sites/utic-platform-test-source",
-        'client_cred': config.client_cred
+        "client_id": config.client_id,
+        "tenant": config.tenant,
+        "site": "https://unstructuredio.sharepoint.com/sites/utic-platform-test-source",
+        "client_cred": config.client_cred,
     }
 
 
 @pytest.fixture
 def indexer_factory(base_sharepoint_config):
     """Factory for creating SharePoint indexers with different configs."""
+
     def _create_indexer(client_cred=None, site=None, path=None):
         access_config = SharepointAccessConfig(
-            client_cred=client_cred or base_sharepoint_config['client_cred']
+            client_cred=client_cred or base_sharepoint_config["client_cred"]
         )
         connection_config = SharepointConnectionConfig(
-            client_id=base_sharepoint_config['client_id'],
-            site=site or base_sharepoint_config['site'],
-            tenant=base_sharepoint_config['tenant'],
+            client_id=base_sharepoint_config["client_id"],
+            site=site or base_sharepoint_config["site"],
+            tenant=base_sharepoint_config["tenant"],
             access_config=access_config,
         )
         index_config = SharepointIndexerConfig(path=path or "")
         return SharepointIndexer(
             connection_config=connection_config,
             index_config=index_config,
         )
+
     return _create_indexer
 
 
 @pytest.mark.asyncio
 @pytest.mark.tags(CONNECTOR_TYPE, SOURCE_TAG, BLOB_STORAGE_TAG)
 @requires_env("SHAREPOINT_CLIENT_ID", "SHAREPOINT_CRED", "MS_TENANT_ID", "MS_USER_PNAME")
-@pytest.mark.parametrize("error_scenario,expected_error", [
-    ("invalid_creds", "UserAuthError"),
-    ("nonexistent_site", "UserError"),
-    ("invalid_path", "UserError"),
-])
+@pytest.mark.parametrize(
+    "error_scenario,expected_error",
+    [
+        ("invalid_creds", "UserAuthError"),
+        ("nonexistent_site", "UserError"),
+        ("invalid_path", "UserError"),
+    ],
+)
 async def test_sharepoint_precheck_error_scenarios(indexer_factory, error_scenario, expected_error):
     """Parametrized test for different SharePoint precheck error scenarios."""
     from unstructured_ingest.errors_v2 import UserAuthError, UserError
-    
-    error_class_map = {
-        "UserAuthError": UserAuthError,
-        "UserError": UserError
-    }
-    
+
+    error_class_map = {"UserAuthError": UserAuthError, "UserError": UserError}
+
     expected_exception = error_class_map[expected_error]
-    
+
     if error_scenario == "invalid_creds":
         indexer = indexer_factory(client_cred="invalid_creds")
     elif error_scenario == "nonexistent_site":
-        indexer = indexer_factory(site="https://unstructuredai.sharepoint.com/sites/definitely-does-not-exist-12345")
+        indexer = indexer_factory(
+            site="https://unstructuredai.sharepoint.com/sites/definitely-does-not-exist-12345"
+        )
     elif error_scenario == "invalid_path":
         indexer = indexer_factory(path="NonExistentFolder/SubFolder/DoesNotExist")
     else:
         pytest.fail(f"Unknown error scenario: {error_scenario}")
-    
+
     with pytest.raises(expected_exception):
         indexer.precheck()
 
@@ -397,21 +401,21 @@ async def test_sharepoint_precheck_error_scenarios(indexer_factory, error_scenar
 def insufficient_perms_config():
     """Config for testing insufficient permissions."""
     from types import SimpleNamespace
-    
+
     required_vars = [
         "SHAREPOINT_CLIENT_ID_INSUFFICIENT",
-        "SHAREPOINT_CRED_INSUFFICIENT", 
-        "MS_TENANT_ID_INSUFFICIENT"
+        "SHAREPOINT_CRED_INSUFFICIENT",
+        "MS_TENANT_ID_INSUFFICIENT",
     ]
-    
+
     missing_vars = [var for var in required_vars if var not in os.environ]
     if missing_vars:
         pytest.skip(f"Missing environment variables: {missing_vars}")
-    
+
     return SimpleNamespace(
         client_id=os.environ["SHAREPOINT_CLIENT_ID_INSUFFICIENT"],
         client_cred=os.environ["SHAREPOINT_CRED_INSUFFICIENT"],
-        tenant=os.environ["MS_TENANT_ID_INSUFFICIENT"]
+        tenant=os.environ["MS_TENANT_ID_INSUFFICIENT"],
     )
 
 
@@ -428,7 +432,7 @@ async def test_sharepoint_precheck_insufficient_permissions(
     access_config = SharepointAccessConfig(client_cred=insufficient_perms_config.client_cred)
     connection_config = SharepointConnectionConfig(
         client_id=insufficient_perms_config.client_id,
-        site=base_sharepoint_config['site'],
+        site=base_sharepoint_config["site"],
         tenant=insufficient_perms_config.tenant,
         access_config=access_config,
     )
@@ -441,5 +445,3 @@ async def test_sharepoint_precheck_insufficient_permissions(
 
     with pytest.raises(UserAuthError):
         indexer.precheck()
-
-

unstructured_ingest/__version__.py

@@ -1 +1 @@
-__version__ = "1.0.57"  # pragma: no cover
+__version__ = "1.0.58"  # pragma: no cover

unstructured_ingest/interfaces/connector.py

@@ -5,6 +5,8 @@
 from pydantic import BaseModel, Secret, model_validator
 from pydantic.types import _SecretBase
 
+from unstructured_ingest.processes.utils.logging.connector import ConnectorLoggingMixin
+
 
 class AccessConfig(BaseModel):
     """Meant to designate holding any sensitive information associated with other configs
@@ -46,5 +48,9 @@ def _is_access_config_optional(self) -> bool:
 
 
 @dataclass
-class BaseConnector(ABC):
+class BaseConnector(ABC, ConnectorLoggingMixin):
     connection_config: ConnectionConfigT
+
+    def __post_init__(self):
+        """Initialize the logging mixin after dataclass initialization."""
+        ConnectorLoggingMixin.__init__(self)