bugfix/add missing lancedb extra (#264)

* add missing extra

* fix text

* Add cloud connector for lancedb

* fix int test

Author: rbiseck3

CHANGELOG.md

@@ -1,10 +1,13 @@
-## 0.3.2-dev4
+## 0.3.3-dev1
 
 ### Enhancements
+
 * **Add `precheck` to Milvus connector**
 
 ### Fixes
-* **Make AstraDB uploader truncate `text` and `text_as_html` content to max 8000 bytes** 
+
+* **Make AstraDB uploader truncate `text` and `text_as_html` content to max 8000 bytes**
+* **Add missing LanceDb extra**
 
 ## 0.3.2
 

setup.py

@@ -1,7 +1,7 @@
 """
 setup.py
 
-unstructured - pre-processing tools for unstructured data
+unstructured-ingest - pre-processing tools for unstructured data
 
 Copyright 2022 Unstructured Technologies, Inc.
 
@@ -104,6 +104,7 @@ def load_requirements(file: Union[str, Path]) -> List[str]:
     "jira": load_requirements("requirements/connectors/jira.in"),
     "kafka": load_requirements("requirements/connectors/kafka.in"),
     "kdbai": load_requirements("requirements/connectors/kdbai.in"),
+    "lancedb": load_requirements("requirements/connectors/lancedb.in"),
     "milvus": load_requirements("requirements/connectors/milvus.in"),
     "mongodb": load_requirements("requirements/connectors/mongodb.in"),
     "notion": load_requirements("requirements/connectors/notion.in"),

test/integration/connectors/test_lancedb.py

@@ -14,9 +14,9 @@
 from test.integration.connectors.utils.constants import DESTINATION_TAG
 from unstructured_ingest.v2.interfaces.file_data import FileData, SourceIdentifiers
 from unstructured_ingest.v2.processes.connectors.lancedb.aws import (
-    LanceDBS3AccessConfig,
-    LanceDBS3ConnectionConfig,
-    LanceDBS3Uploader,
+    LanceDBAwsAccessConfig,
+    LanceDBAwsConnectionConfig,
+    LanceDBAwsUploader,
 )
 from unstructured_ingest.v2.processes.connectors.lancedb.azure import (
     LanceDBAzureAccessConfig,
@@ -156,7 +156,7 @@ def _get_uri(target: Literal["local", "s3", "gcs", "az"], local_base_path: Path)
 
 def _get_uploader(
     uri: str,
-) -> Union[LanceDBAzureUploader, LanceDBAzureUploader, LanceDBS3Uploader, LanceDBGSPUploader]:
+) -> Union[LanceDBAzureUploader, LanceDBAzureUploader, LanceDBAwsUploader, LanceDBGSPUploader]:
     target = uri.split("://", maxsplit=1)[0] if uri.startswith(("s3", "az", "gs")) else "local"
     if target == "az":
         azure_connection_string = os.getenv("AZURE_DEST_CONNECTION_STR")
@@ -170,10 +170,10 @@ def _get_uploader(
         )
 
     elif target == "s3":
-        return LanceDBS3Uploader(
+        return LanceDBAwsUploader(
             upload_config=LanceDBUploaderConfig(table_name=TABLE_NAME),
-            connection_config=LanceDBS3ConnectionConfig(
-                access_config=LanceDBS3AccessConfig(
+            connection_config=LanceDBAwsConnectionConfig(
+                access_config=LanceDBAwsAccessConfig(
                     aws_access_key_id=os.getenv("S3_INGEST_TEST_ACCESS_KEY"),
                     aws_secret_access_key=os.getenv("S3_INGEST_TEST_SECRET_KEY"),
                 ),

unstructured_ingest/__version__.py

@@ -1 +1 @@
-__version__ = "0.3.2-dev4"  # pragma: no cover
+__version__ = "0.3.3-dev1"  # pragma: no cover

unstructured_ingest/v2/processes/connectors/lancedb/__init__.py

@@ -6,12 +6,25 @@
 from .aws import lancedb_aws_destination_entry
 from .azure import CONNECTOR_TYPE as LANCEDB_AZURE_CONNECTOR_TYPE
 from .azure import lancedb_azure_destination_entry
+from .cloud import CONNECTOR_TYPE as LANCEDB_CLOUD_CONNECTOR_TYPE
+from .cloud import lancedb_cloud_destination_entry
 from .gcp import CONNECTOR_TYPE as LANCEDB_GCS_CONNECTOR_TYPE
 from .gcp import lancedb_gcp_destination_entry
 from .local import CONNECTOR_TYPE as LANCEDB_LOCAL_CONNECTOR_TYPE
 from .local import lancedb_local_destination_entry
 
-add_destination_entry(LANCEDB_S3_CONNECTOR_TYPE, lancedb_aws_destination_entry)
-add_destination_entry(LANCEDB_AZURE_CONNECTOR_TYPE, lancedb_azure_destination_entry)
-add_destination_entry(LANCEDB_GCS_CONNECTOR_TYPE, lancedb_gcp_destination_entry)
-add_destination_entry(LANCEDB_LOCAL_CONNECTOR_TYPE, lancedb_local_destination_entry)
+add_destination_entry(
+    destination_type=LANCEDB_S3_CONNECTOR_TYPE, entry=lancedb_aws_destination_entry
+)
+add_destination_entry(
+    destination_type=LANCEDB_AZURE_CONNECTOR_TYPE, entry=lancedb_azure_destination_entry
+)
+add_destination_entry(
+    destination_type=LANCEDB_GCS_CONNECTOR_TYPE, entry=lancedb_gcp_destination_entry
+)
+add_destination_entry(
+    destination_type=LANCEDB_LOCAL_CONNECTOR_TYPE, entry=lancedb_local_destination_entry
+)
+add_destination_entry(
+    destination_type=LANCEDB_CLOUD_CONNECTOR_TYPE, entry=lancedb_cloud_destination_entry
+)

unstructured_ingest/v2/processes/connectors/lancedb/aws.py

@@ -15,28 +15,28 @@
 CONNECTOR_TYPE = "lancedb_aws"
 
 
-class LanceDBS3AccessConfig(AccessConfig):
+class LanceDBAwsAccessConfig(AccessConfig):
     aws_access_key_id: str = Field(description="The AWS access key ID to use.")
     aws_secret_access_key: str = Field(description="The AWS secret access key to use.")
 
 
-class LanceDBS3ConnectionConfig(LanceDBRemoteConnectionConfig):
-    access_config: Secret[LanceDBS3AccessConfig]
+class LanceDBAwsConnectionConfig(LanceDBRemoteConnectionConfig):
+    access_config: Secret[LanceDBAwsAccessConfig]
 
     def get_storage_options(self) -> dict:
         return {**self.access_config.get_secret_value().model_dump(), "timeout": self.timeout}
 
 
 @dataclass
-class LanceDBS3Uploader(LanceDBUploader):
+class LanceDBAwsUploader(LanceDBUploader):
     upload_config: LanceDBUploaderConfig
-    connection_config: LanceDBS3ConnectionConfig
+    connection_config: LanceDBAwsConnectionConfig
     connector_type: str = CONNECTOR_TYPE
 
 
 lancedb_aws_destination_entry = DestinationRegistryEntry(
-    connection_config=LanceDBS3ConnectionConfig,
-    uploader=LanceDBS3Uploader,
+    connection_config=LanceDBAwsConnectionConfig,
+    uploader=LanceDBAwsUploader,
     uploader_config=LanceDBUploaderConfig,
     upload_stager_config=LanceDBUploadStagerConfig,
     upload_stager=LanceDBUploadStager,

unstructured_ingest/v2/processes/connectors/lancedb/cloud.py

@@ -0,0 +1,42 @@
+from dataclasses import dataclass
+
+from pydantic import Field, Secret
+
+from unstructured_ingest.v2.interfaces.connector import AccessConfig
+from unstructured_ingest.v2.processes.connector_registry import DestinationRegistryEntry
+from unstructured_ingest.v2.processes.connectors.lancedb.lancedb import (
+    LanceDBRemoteConnectionConfig,
+    LanceDBUploader,
+    LanceDBUploaderConfig,
+    LanceDBUploadStager,
+    LanceDBUploadStagerConfig,
+)
+
+CONNECTOR_TYPE = "lancedb_cloud"
+
+
+class LanceDBCloudAccessConfig(AccessConfig):
+    api_key: str = Field(description="Api key associated with LanceDb cloud")
+
+
+class LanceDBCloudConnectionConfig(LanceDBRemoteConnectionConfig):
+    access_config: Secret[LanceDBCloudAccessConfig]
+
+    def get_storage_options(self) -> dict:
+        return {**self.access_config.get_secret_value().model_dump(), "timeout": self.timeout}
+
+
+@dataclass
+class LanceDBCloudUploader(LanceDBUploader):
+    upload_config: LanceDBUploaderConfig
+    connection_config: LanceDBCloudConnectionConfig
+    connector_type: str = CONNECTOR_TYPE
+
+
+lancedb_cloud_destination_entry = DestinationRegistryEntry(
+    connection_config=LanceDBCloudConnectionConfig,
+    uploader=LanceDBCloudUploader,
+    uploader_config=LanceDBUploaderConfig,
+    upload_stager_config=LanceDBUploadStagerConfig,
+    upload_stager=LanceDBUploadStager,
+)