fix: unable to convert DeltaTableConnectionConfig to json schema (#566)

A bump to the pydantic library now requires typing for Secrets. As a
result, In its current form, running

```python
from unstructured_ingest.processes.connectors.delta_table import DeltaTableConnectionConfig
DeltaTableConnectionConfig(table_uri="").model_json_schema()
```

results in
```
    raise TypeError(
TypeError: Can't get secret type from Secret. Please use Secret[<type>], or subclass from Secret[<type>] instead.
```

In addition to the fix, this PR adds tests to iterate over all
connectors and validate that the json schema can be generated from them.

Author: ryannikolaidis

CHANGELOG.md

@@ -1,3 +1,7 @@
+## 1.1.2
+
+* **Fix**: DeltaTableConnectionConfig default assignment is compliant with stricter typing in Pydantic
+
 ## 1.1.1
 
 * **Fix: Update examples**

test/unit/connectors/test_json_schema.py

@@ -0,0 +1,62 @@
+import pytest
+
+from unstructured_ingest.processes.connector_registry import (
+    destination_registry,
+    source_registry,
+)
+
+
+def test_all_source_connector_configs_json_schema():
+    failed_connectors = []
+
+    for connector_type, connector_entry in source_registry.items():
+        try:
+            if connector_entry.connection_config:
+                schema = connector_entry.connection_config.model_json_schema()
+                assert schema is not None, f"Schema is None for {connector_type} connection_config"
+
+            if connector_entry.indexer_config:
+                schema = connector_entry.indexer_config.model_json_schema()
+                assert schema is not None, f"Schema is None for {connector_type} indexer_config"
+
+            if connector_entry.downloader_config:
+                schema = connector_entry.downloader_config.model_json_schema()
+                assert schema is not None, f"Schema is None for {connector_type} downloader_config"
+
+        except Exception as e:
+            failed_connectors.append((connector_type, str(e)))
+
+    if failed_connectors:
+        error_msg = "Failed to generate JSON schema for source connectors:\n"
+        for connector_type, error in failed_connectors:
+            error_msg += f"  - {connector_type}: {error}\n"
+        pytest.fail(error_msg)
+
+
+def test_all_destination_connector_configs_json_schema():
+    failed_connectors = []
+
+    for connector_type, connector_entry in destination_registry.items():
+        try:
+            if connector_entry.connection_config:
+                schema = connector_entry.connection_config.model_json_schema()
+                assert schema is not None, f"Schema is None for {connector_type} connection_config"
+
+            if connector_entry.uploader_config:
+                schema = connector_entry.uploader_config.model_json_schema()
+                assert schema is not None, f"Schema is None for {connector_type} uploader_config"
+
+            if connector_entry.upload_stager_config:
+                schema = connector_entry.upload_stager_config.model_json_schema()
+                assert schema is not None, (
+                    f"Schema is None for {connector_type} upload_stager_config"
+                )
+
+        except Exception as e:
+            failed_connectors.append((connector_type, str(e)))
+
+    if failed_connectors:
+        error_msg = "Failed to generate JSON schema for destination connectors:\n"
+        for connector_type, error in failed_connectors:
+            error_msg += f"  - {connector_type}: {error}\n"
+        pytest.fail(error_msg)

unstructured_ingest/__version__.py

@@ -1 +1 @@
-__version__ = "1.1.1"  # pragma: no cover
+__version__ = "1.1.2"  # pragma: no cover

unstructured_ingest/processes/connectors/delta_table.py

@@ -48,7 +48,8 @@ class DeltaTableAccessConfig(AccessConfig):
 
 class DeltaTableConnectionConfig(ConnectionConfig):
     access_config: Secret[DeltaTableAccessConfig] = Field(
-        default=Secret(DeltaTableAccessConfig()), validate_default=True
+        default_factory=lambda: Secret[DeltaTableAccessConfig](DeltaTableAccessConfig()),
+        validate_default=True,
     )
     aws_region: Optional[str] = Field(default=None, description="AWS Region")
     table_uri: str = Field(