feat/use uuid for s3 identifiers (#8)

* use uuid for s3 identifiers

* Fix mapping between indexer and downloader

Author: rbiseck3

CHANGELOG.md

@@ -1,3 +1,9 @@
+## 0.0.2-dev0
+
+### Enhancements
+
+* **Use uuid for s3 identifiers** Update unique id to use uuid derived from file path rather than the filepath itself.
+
 ## 0.0.1
 
 ### Enhancements

scripts/version-sync.sh

@@ -106,7 +106,7 @@ fi
 declare FAILED_CHECK=0
 
 git fetch origin main
-MAIN_VERSION=$(git show origin/main:unstructured/__version__.py | grep -o -m 1 -E "${RE_SEMVER_FULL}")
+MAIN_VERSION=$(git show origin/main:unstructured_ingest/__version__.py | grep -o -m 1 -E "${RE_SEMVER_FULL}")
 MAIN_IS_RELEASE=false
 [[ $MAIN_VERSION != *"-dev"* ]] && MAIN_IS_RELEASE=true
 CURRENT_BRANCH=$(git rev-parse --abbrev-ref HEAD)

unstructured_ingest/__version__.py

@@ -1 +1 @@
-__version__ = "0.0.1"  # pragma: no cover
+__version__ = "0.0.2-dev0"  # pragma: no cover

unstructured_ingest/v2/examples/example_s3.py

@@ -24,7 +24,7 @@
 if __name__ == "__main__":
     logger.info(f"Writing all content in: {work_dir.resolve()}")
     Pipeline.from_configs(
-        context=ProcessorConfig(work_dir=str(work_dir.resolve())),
+        context=ProcessorConfig(work_dir=str(work_dir.resolve()), verbose=True),
         indexer_config=S3IndexerConfig(remote_url="s3://utic-dev-tech-fixtures/small-pdf-set/"),
         downloader_config=S3DownloaderConfig(download_dir=download_path),
         source_connection_config=S3ConnectionConfig(anonymous=True),

unstructured_ingest/v2/processes/connectors/fsspec/fsspec.py

@@ -7,6 +7,7 @@
 from pathlib import Path
 from time import time
 from typing import TYPE_CHECKING, Any, Generator, Optional, TypeVar
+from uuid import NAMESPACE_DNS, uuid5
 
 from unstructured.documents.elements import DataSourceMetadata
 
@@ -210,16 +211,19 @@ def run(self, **kwargs: Any) -> Generator[FileData, None, None]:
             # Note: we remove any remaining leading slashes (Box introduces these)
             # to get a valid relative path
             rel_path = file.replace(self.index_config.path_without_protocol, "").lstrip("/")
+
+            additional_metadata = self.sterilize_info(path=file)
+            additional_metadata["original_file_path"] = file
             yield FileData(
-                identifier=file,
+                identifier=str(uuid5(NAMESPACE_DNS, file)),
                 connector_type=self.connector_type,
                 source_identifiers=SourceIdentifiers(
                     filename=Path(file).name,
                     rel_path=rel_path or None,
                     fullpath=file,
                 ),
                 metadata=self.get_metadata(path=file),
-                additional_metadata=self.sterilize_info(path=file),
+                additional_metadata=additional_metadata,
             )
 
 
@@ -262,7 +266,8 @@ def run(self, file_data: FileData, **kwargs: Any) -> DownloadResponse:
         download_path = self.get_download_path(file_data=file_data)
         download_path.parent.mkdir(parents=True, exist_ok=True)
         try:
-            self.fs.get(rpath=file_data.identifier, lpath=download_path.as_posix())
+            rpath = file_data.additional_metadata["original_file_path"]
+            self.fs.get(rpath=rpath, lpath=download_path.as_posix())
         except Exception as e:
             logger.error(f"failed to download file {file_data.identifier}: {e}", exc_info=True)
             raise SourceConnectionNetworkError(f"failed to download file {file_data.identifier}")
@@ -272,7 +277,8 @@ async def async_run(self, file_data: FileData, **kwargs: Any) -> DownloadRespons
         download_path = self.get_download_path(file_data=file_data)
         download_path.parent.mkdir(parents=True, exist_ok=True)
         try:
-            await self.fs.get(rpath=file_data.identifier, lpath=download_path.as_posix())
+            rpath = file_data.additional_metadata["original_file_path"]
+            await self.fs.get(rpath=rpath, lpath=download_path.as_posix())
         except Exception as e:
             logger.error(f"failed to download file {file_data.identifier}: {e}", exc_info=True)
             raise SourceConnectionNetworkError(f"failed to download file {file_data.identifier}")