fix: handle empty buckets (#152)

* fix: handle empty buckets

* fix:update json file with new page response with no CommonPrefixes; updateunit tes with new expected empty list

* fix: remove NextContinuationToken and set IsTruncated to false

Author: ajar312

.flake8

@@ -0,0 +1,11 @@
+[flake8]
+
+exclude =
+    .git,
+    __pycache__,
+    build,
+    .venv,
+    venv, 
+    .conda
+max-complexity = 10
+ 

src/aind_data_asset_indexer/utils.py

@@ -452,7 +452,7 @@ def iterate_through_top_level(
     for page in pages:
         yield [
             p.get("Prefix")
-            for p in page["CommonPrefixes"]
+            for p in page.get("CommonPrefixes", [])
             if p.get("Prefix") is not None
         ]
 

tests/resources/utils/example_pages_response.json

@@ -101,5 +101,29 @@
       "EncodingType": "url",
       "KeyCount": 1,
       "ContinuationToken": "1GQ5"
+   },
+    {
+      "ResponseMetadata": {
+         "RequestId": "RequestId",
+         "HostId": "HostId",
+         "HTTPStatusCode": 200,
+         "HTTPHeaders": {
+            "x-amz-id-2": "x-amz-id-2",
+            "x-amz-request-id": "x-amz-request-id",
+            "date": "Mon, 13 May 2024 21:22:24 GMT",
+            "x-amz-bucket-region": "us-west-2",
+            "content-type": "application/xml",
+            "transfer-encoding": "chunked",
+            "server": "AmazonS3"
+         },
+         "RetryAttempts": 0
+      },
+      "IsTruncated": false,
+      "Name": "aind-ephys-data-dev-u5u0i5",
+      "Prefix": "",
+      "Delimiter": "/",
+      "MaxKeys": 3,
+      "EncodingType": "url",
+      "KeyCount": 3
    }
 ]

tests/test_utils.py

@@ -589,6 +589,7 @@ def test_iterate_through_top_level(self, mock_s3_client):
                 "ecephys_655019_2000-01-01_01-01-02/",
             ],
             ["ecephys_684156_2000-01-01_00-00-01/"],
+            [],
         ]
 
         self.assertEqual(expected_output, list(output))