Merge pull request #179 from AllenNeuralDynamics/release-v0.21.1

Release v0.21.1

Author: helen-m-lin

src/aind_data_asset_indexer/__init__.py

@@ -1,3 +1,3 @@
 """Package"""
 
-__version__ = "0.21.0"
+__version__ = "0.21.1"

src/aind_data_asset_indexer/codeocean_bucket_indexer.py

@@ -164,11 +164,16 @@ def _get_co_links_from_record(docdb_record: dict) -> List[str]:
             external_links = external_links.get(
                 ExternalPlatforms.CODEOCEAN.value, []
             )
-        else:
+        elif isinstance(external_links, list):
+            if not all(isinstance(r, dict) for r in external_links):
+                raise ValueError(f"Invalid external_links for: {docdb_record}")
             external_links = [
                 r.get(ExternalPlatforms.CODEOCEAN.value)
                 for r in external_links
+                if r.get(ExternalPlatforms.CODEOCEAN.value) is not None
             ]
+        else:
+            raise ValueError(f"Invalid external_links for: {docdb_record}")
         return external_links
 
     def _update_external_links_in_docdb(
@@ -213,37 +218,45 @@ def _update_external_links_in_docdb(
             for page in pages:
                 records_to_update = []
                 for record in page:
-                    location = record.get("location")
-                    external_links = self._get_co_links_from_record(record)
-                    code_ocean_ids = (
-                        None
-                        if location is None
-                        else co_loc_to_id_map.get(location)
-                    )
-                    docdb_rec_id = record["_id"]
-                    if code_ocean_ids is not None and code_ocean_ids != set(
-                        external_links
-                    ):
-                        new_external_links = code_ocean_ids
-                    elif external_links and not code_ocean_ids:
-                        logging.info(
-                            f"No code ocean data asset ids found for "
-                            f"{location}. Removing external links from record."
+                    try:
+                        location = record.get("location")
+                        external_links = self._get_co_links_from_record(record)
+                        code_ocean_ids = (
+                            None
+                            if location is None
+                            else co_loc_to_id_map.get(location)
                         )
-                        new_external_links = set()
-                    else:
-                        new_external_links = None
-                    if new_external_links is not None:
-                        record_links = {
-                            ExternalPlatforms.CODEOCEAN.value: sorted(
-                                list(new_external_links)
+                        docdb_rec_id = record["_id"]
+                        if (
+                            code_ocean_ids is not None
+                            and code_ocean_ids != set(external_links)
+                        ):
+                            new_external_links = code_ocean_ids
+                        elif external_links and not code_ocean_ids:
+                            logging.info(
+                                f"No code ocean data asset ids found for "
+                                f"{location}. Removing external links from "
+                                "record."
                             )
-                        }
-                        records_to_update.append(
-                            {
-                                "_id": docdb_rec_id,
-                                "external_links": record_links,
+                            new_external_links = set()
+                        else:
+                            new_external_links = None
+                        if new_external_links is not None:
+                            record_links = {
+                                ExternalPlatforms.CODEOCEAN.value: sorted(
+                                    list(new_external_links)
+                                )
                             }
+                            records_to_update.append(
+                                {
+                                    "_id": docdb_rec_id,
+                                    "external_links": record_links,
+                                }
+                            )
+                    except Exception as e:
+                        logging.error(
+                            f'Error processing {record.get("location")}: '
+                            f"{repr(e)}"
                         )
                 if len(records_to_update) > 0:
                     logging.info(f"Updating {len(records_to_update)} records")

tests/test_codeocean_bucket_indexer.py

@@ -207,6 +207,32 @@ def test_get_co_links_from_record_legacy(self):
         )
         self.assertEqual(["abc-123", "def-456"], output)
 
+    def test_get_co_links_from_record_invalid_list(self):
+        """Tests _get_co_links_from_record with invalid external_links list"""
+        docdb_record = {
+            "_id": "12345",
+            "location": "s3://bucket/prefix",
+            "external_links": ["abc-123", "def-456"],
+        }
+        with self.assertRaises(ValueError) as e:
+            self.basic_job._get_co_links_from_record(docdb_record=docdb_record)
+        self.assertEqual(
+            f"Invalid external_links for: {docdb_record}", str(e.exception)
+        )
+
+    def test_get_co_links_from_record_invalid_other(self):
+        """Tests _get_co_links_from_record with invalid external_links type"""
+        docdb_record = {
+            "_id": "12345",
+            "location": "s3://bucket/prefix",
+            "external_links": "abc-123",
+        }
+        with self.assertRaises(ValueError) as e:
+            self.basic_job._get_co_links_from_record(docdb_record=docdb_record)
+        self.assertEqual(
+            f"Invalid external_links for: {docdb_record}", str(e.exception)
+        )
+
     @patch("aind_data_asset_indexer.codeocean_bucket_indexer.MetadataDbClient")
     @patch("codeocean.data_asset.DataAssets.search_data_assets_iterator")
     @patch("aind_data_asset_indexer.codeocean_bucket_indexer.paginate_docdb")
@@ -262,6 +288,11 @@ def test_update_external_links_in_docdb(
                     "location": "s3://bucket2/prefix4",
                     "external_links": [],
                 },
+                {
+                    "_id": "0004",
+                    "location": "s3://bucket3/prefix5",
+                    "external_links": ["def-456"],
+                },
             ]
         ]
 
@@ -273,6 +304,10 @@ def test_update_external_links_in_docdb(
         expected_log_messages = [
             "INFO:root:No code ocean data asset ids found for "
             "s3://bucket2/prefix3. Removing external links from record.",
+            "ERROR:root:Error processing s3://bucket3/prefix5: "
+            "ValueError(\"Invalid external_links for: {'_id': '0004', "
+            "'location': 's3://bucket3/prefix5', 'external_links': "
+            "['def-456']}\")",
             "INFO:root:Updating 2 records",
             f"DEBUG:root:[{bulk_write_response}]",
         ]