fix: Weaviate Collection naming conventions fix (#489)

* Fix Weaviate uploader connection naming mechanism

* Integration test weaviate collection name

* Update changelog and version; weaviate collection names fix

Author: mpolomdeepsense

CHANGELOG.md

@@ -1,3 +1,7 @@
+## 1.0.20
+
+* **Fix Weaviate connector issue with names being wrongly transformed to match collections naming conventions**
+
 ## 1.0.19
 
 * **Fix databricks delta table name edge cases**

test/integration/connectors/weaviate/test_local.py

@@ -141,8 +141,8 @@ def test_weaviate_local_create_destination(weaviate_instance):
         upload_config=LocalWeaviateUploaderConfig(),
         connection_config=LocalWeaviateConnectionConfig(),
     )
-    collection_name = "system_created-123"
-    formatted_collection_name = "System_created_123"
+    collection_name = "system_CREATED-123"
+    formatted_collection_name = "System_CREATED_123"
     created = uploader.create_destination(destination_name=collection_name)
     assert created
     with uploader.connection_config.get_client() as weaviate_client:

test/unit/connectors/weaviate/test_weaviate.py

@@ -0,0 +1,105 @@
+from unittest.mock import MagicMock
+
+import pytest
+from pydantic import Secret
+
+from unstructured_ingest.processes.connectors.weaviate.weaviate import (
+    WeaviateAccessConfig,
+    WeaviateConnectionConfig,
+    WeaviateUploader,
+    WeaviateUploaderConfig,
+)
+
+
+class WeaviateConnectionConfigTest(WeaviateConnectionConfig):
+    def get_client(self):
+        yield MagicMock()
+
+
+@pytest.fixture
+def access_config():
+    return WeaviateAccessConfig()
+
+
+@pytest.fixture
+def connection_config(access_config: WeaviateAccessConfig):
+    return WeaviateConnectionConfigTest(
+        access_config=Secret(access_config),
+        init_timeout=10,
+        query_timeout=10,
+        insert_timeout=10,
+    )
+
+
+@pytest.fixture
+def uploader_config():
+    return WeaviateUploaderConfig(collection=None)
+
+
+@pytest.fixture
+def uploader(
+    connection_config: WeaviateConnectionConfigTest, uploader_config: WeaviateUploaderConfig
+):
+    return WeaviateUploader(
+        connection_config=connection_config,
+        upload_config=uploader_config,
+        connector_type="weaviate",
+    )
+
+
+@pytest.mark.parametrize(
+    ("destination_name", "expected"),
+    [
+        ("t", "T"),
+        ("test123", "Test123"),
+        ("test__ __", "Test_____"),
+        ("test-Name", "Test_Name"),
+        ("teSt name", "TeSt_name"),
+        ("test@name#123", "Test_name_123"),
+    ],
+)
+def test_format_destination_name_success_logs(
+    caplog: pytest.LogCaptureFixture,
+    uploader: WeaviateUploader,
+    destination_name: str,
+    expected: str,
+):
+    formatted_name = uploader.format_destination_name(destination_name)
+    assert formatted_name == expected
+    assert len(caplog.records) == 1
+    assert (
+        f"Given Collection name '{destination_name}' doesn't follow naming conventions. "
+        f"Renaming to '{expected}'"
+    ) in caplog.text
+
+
+@pytest.mark.parametrize(
+    ("destination_name", "expected"),
+    [
+        ("T", "T"),
+        ("Test_123", "Test_123"),
+        ("TEST_NAME", "TEST_NAME"),
+    ],
+)
+def test_format_destination_name_success_no_logs(
+    caplog: pytest.LogCaptureFixture,
+    uploader: WeaviateUploader,
+    destination_name: str,
+    expected: str,
+):
+    formatted_name = uploader.format_destination_name(destination_name)
+    assert formatted_name == expected
+    assert len(caplog.records) == 0
+
+
+@pytest.mark.parametrize(
+    ("destination_name"),
+    [
+        ("123name"),
+        ("@#$%^&*"),
+        (""),
+    ],
+)
+def test_format_destination_name_error(uploader: WeaviateUploader, destination_name: str):
+    with pytest.raises(ValueError):
+        uploader.format_destination_name(destination_name)

unstructured_ingest/__version__.py

@@ -1 +1 @@
-__version__ = "1.0.19"  # pragma: no cover
+__version__ = "1.0.20"  # pragma: no cover

unstructured_ingest/processes/connectors/weaviate/weaviate.py

@@ -234,34 +234,51 @@ def init(self, **kwargs: Any) -> None:
         self.create_destination(**kwargs)
 
     def format_destination_name(self, destination_name: str) -> str:
-        # Weaviate naming requirements:
-        # must be alphanumeric and underscores only
+        """
+        Weaviate Collection naming conventions:
+            1. must begin with an uppercase letter
+            2. must be alphanumeric and underscores only
+        """
+
+        # Check if the first character is an uppercase letter
+        if not re.match(r"^[a-zA-Z]", destination_name):
+            raise ValueError("Collection name must start with an uppercase letter")
+        # Replace all non-alphanumeric characters with underscores
         formatted = re.sub(r"[^a-zA-Z0-9]", "_", destination_name)
-        # must begin with capital letter
-        return formatted.capitalize()
+        # Make the first character uppercase and leave the rest as is
+        if len(formatted) == 1:
+            formatted = formatted.capitalize()
+        else:
+            formatted = formatted[0].capitalize() + formatted[1:]
+        if formatted != destination_name:
+            logger.warning(
+                f"Given Collection name '{destination_name}' doesn't follow naming conventions. "
+                f"Renaming to '{formatted}'"
+            )
+        return formatted
 
     def create_destination(
         self,
-        destination_name: str = "unstructuredautocreated",
+        destination_name: str = "Unstructuredautocreated",
         vector_length: Optional[int] = None,
         **kwargs: Any,
     ) -> bool:
         collection_name = self.upload_config.collection or destination_name
         collection_name = self.format_destination_name(collection_name)
         self.upload_config.collection = collection_name
 
-        connectors_dir = Path(__file__).parents[1]
-        collection_config_file = connectors_dir / "assets" / "weaviate_collection_config.json"
-        with collection_config_file.open() as f:
-            collection_config = json.load(f)
-        collection_config["class"] = collection_name
-
         if not self._collection_exists():
-            logger.info(f"creating weaviate collection '{collection_name}' with default configs")
+            connectors_dir = Path(__file__).parents[1]
+            collection_config_file = connectors_dir / "assets" / "weaviate_collection_config.json"
+            with collection_config_file.open() as f:
+                collection_config = json.load(f)
+            collection_config["class"] = collection_name
+
+            logger.info(f"Creating weaviate collection '{collection_name}' with default configs")
             with self.connection_config.get_client() as weaviate_client:
                 weaviate_client.collections.create_from_dict(config=collection_config)
                 return True
-        logger.debug(f"collection with name '{collection_name}' already exists, skipping creation")
+        logger.debug(f"Collection with name '{collection_name}' already exists, skipping creation")
         return False
 
     def check_for_errors(self, client: "WeaviateClient") -> None: