Fix cap redis version (#519)

- cap redis client version to avoid breaking uploader plugin
- fixed azure and s3 e2e test script so that they no longer report error

Author: bryan-unstructured

CHANGELOG.md

@@ -1,3 +1,7 @@
+## 1.0.31
+
+* **Cap redis client version to 5.3.0**
+
 ## 1.0.30
 
 * **Fixed issue in the blob storage destination connector where files with the same name were overwriting each other**

requirements/connectors/redis.txt

@@ -1 +1 @@
-redis
+redis<=5.3.0

test_e2e/dest/s3.sh

@@ -49,7 +49,8 @@ PYTHONPATH=${PYTHONPATH:-.} "$RUN_SCRIPT" \
 
 # Simply check the number of files uploaded
 expected_num_files=1
-num_files_in_s3=$(AWS_ACCESS_KEY_ID="$S3_INGEST_TEST_ACCESS_KEY" AWS_SECRET_ACCESS_KEY="$S3_INGEST_TEST_SECRET_KEY" aws s3 ls "${DESTINATION_S3}" --region us-east-2 | grep -c "\.json$")
+num_files_in_s3=$(AWS_ACCESS_KEY_ID="$S3_INGEST_TEST_ACCESS_KEY" AWS_SECRET_ACCESS_KEY="$S3_INGEST_TEST_SECRET_KEY" aws s3 ls "${DESTINATION_S3}" --region us-east-2 --recursive | grep -c "\.json$")
+
 if [ "$num_files_in_s3" -ne "$expected_num_files" ]; then
   echo "Expected $expected_num_files files to be uploaded to s3, but found $num_files_in_s3 files."
   exit 1

test_e2e/python/test-azure-output.py

@@ -1,5 +1,4 @@
 #!/usr/bin/env python
-import azure.core.exceptions
 import click
 from azure.storage.blob import ContainerClient
 
@@ -19,14 +18,20 @@ def down(connection_string: str, container: str, blob_path: str):
     )
     blob_list = [b.name for b in list(container_client.list_blobs(name_starts_with=blob_path))]
     print(f"deleting all content from {container}/{blob_path}")
-    # Delete all content in folder first
-    container_client.delete_blobs(*[b for b in blob_list if b != blob_path])
-
-    # Delete folder itself
-    try:
-        container_client.delete_blob(blob_path)
-    except azure.core.exceptions.ResourceNotFoundError:
-        print(f"folder {blob_path} not found")
+
+    files = []
+    folders = []
+    for blob in blob_list:
+        blob_props = container_client.get_blob_client(blob).get_blob_properties()
+        if blob_props.size == 0 and not blob.endswith("/_empty"):
+            folders.append(blob)
+        else:
+            files.append(blob)
+
+    # Delete all content in folders first, then delete the folders
+    container_client.delete_blobs(*files)
+    for folder in folders[::-1]:
+        container_client.delete_blob(folder)
 
 
 @cli.command()

unstructured_ingest/__version__.py

@@ -1 +1 @@
-__version__ = "1.0.30"  # pragma: no cover
+__version__ = "1.0.31"  # pragma: no cover

unstructured_ingest/processes/connectors/fsspec/fsspec.py

@@ -343,9 +343,9 @@ def precheck(self) -> None:
             raise self.wrap_error(e=e)
 
     def get_upload_path(self, file_data: FileData) -> Path:
-        upload_path = (
-            Path(self.upload_config.path_without_protocol) / file_data.source_identifiers.fullpath
-        )
+        upload_path = Path(
+            self.upload_config.path_without_protocol
+        ) / file_data.source_identifiers.fullpath.lstrip("/")
         updated_upload_path = upload_path.parent / f"{upload_path.name}.json"
         return updated_upload_path