Merge pull request #159 from AllenNeuralDynamics/release-v0.19.0

Release v0.19.0

Author: helen-m-lin

docs/source/UserGuide.rst

@@ -62,8 +62,10 @@ The workflow is generally as follows:
 
    -  If the metadata record exists in S3 but not in DocDB, copy it
       to DocDB.
-   -  If the metadata record does not exist in S3, create it and save
-      it to S3. Assume a Lambda function will move it over to DocDB.
+   -  If the metadata record for a derived asset does not exist in S3,
+      create it and save it to S3. Assume a Lambda function will move it
+      over to DocDB. Metadata records for raw assets are created during
+      the upload process, **not** by this job.
    -  In both cases above, ensure the original metadata folder and core
       files are in sync with the metadata.nd.json file.
 

src/aind_data_asset_indexer/__init__.py

@@ -1,3 +1,3 @@
 """Package"""
 
-__version__ = "0.18.1"
+__version__ = "0.19.0"

src/aind_data_asset_indexer/aind_bucket_indexer.py

@@ -19,6 +19,7 @@
     get_s3_location,
     paginate_docdb,
 )
+from aind_data_schema_models.data_name_patterns import DataLevel
 from mypy_boto3_s3 import S3Client
 from mypy_boto3_s3.type_defs import CopySourceTypeDef
 from requests.adapters import HTTPAdapter
@@ -67,8 +68,8 @@ class AindIndexBucketJob:
     4.0) If a metadata record exists, check if it is in DocDB.
     4.1) If already in DocDb, then don't do anything.
     Otherwise, copy record to DocDB.
-    4.2) If a metadata record does not exist, then build one and save it to S3.
-    Assume a lambda function will move it over to DocDb.
+    4.2) If a metadata record does not exist and the asset is derived, then
+    build and save it to S3. Assume a lambda function will move it to DocDB.
     4.3) In both cases above, ensure the original metadata folder and core
     files are in sync with the metadata.nd.json file.
     """
@@ -535,6 +536,38 @@ def _process_records(self, records: List[dict]):
         )
         mapped_partitions.compute()
 
+    def _get_data_level_for_prefix(
+        self, s3_client: S3Client, bucket: str, prefix: str
+    ) -> Optional[str]:
+        """
+        Get an asset's data level from the data_description.json file.
+
+        Parameters
+        ----------
+        s3_client : S3Client
+        bucket : str
+        prefix : str
+
+        Returns
+        -------
+        Optional[str]
+          The data level of the asset. Returns None if data_description.json
+          file is not found.
+        """
+        data_desc_key = create_object_key(
+            prefix=prefix, filename="data_description.json"
+        )
+        if does_s3_object_exist(
+            s3_client=s3_client, bucket=bucket, key=data_desc_key
+        ):
+            json_contents = download_json_file_from_s3(
+                s3_client=s3_client,
+                bucket=bucket,
+                object_key=data_desc_key,
+            )
+            return json_contents.get("data_level") if json_contents else None
+        return None
+
     def _process_prefix(
         self,
         s3_prefix: str,
@@ -548,8 +581,9 @@ def _process_prefix(
         2) If record is in S3 but not DocDb, then copy it to DocDb if the
         location in the metadata record matches the actual location and
         the record has an _id field. Otherwise, log a warning.
-        3) If record does not exist in both DocDB and S3, build a new metadata
-        file and save it to S3 (assume Lambda function will save to DocDB).
+        3) If record does not exist in both DocDB and S3, check the data level.
+        For derived assets, build a new metadata file and save it to S3 (assume
+        Lambda function will save to DocDB).
         4) In both cases above, we also copy the original core json files to a
         subfolder and ensure the top level core jsons are in sync with the
         metadata.nd.json in S3.
@@ -640,7 +674,12 @@ def _process_prefix(
                     f"Metadata record for {s3_full_location} "
                     f"already exists in DocDb. Skipping."
                 )
-        else:  # metadata.nd.json file does not exist in S3. Create a new one.
+        elif (
+            self._get_data_level_for_prefix(
+                s3_client=s3_client, bucket=bucket, prefix=s3_prefix
+            )
+            == DataLevel.DERIVED.value
+        ):
             # Build a new metadata file, save it to S3 and save it to DocDb.
             # Also copy the original core json files to a subfolder and then
             # overwrite them with the new fields from metadata.nd.json.
@@ -675,6 +714,11 @@ def _process_prefix(
                 logging.warning(
                     f"Unable to build metadata record for: {location}!"
                 )
+        else:
+            logging.info(
+                f"Metadata record for {location} not found in S3 and data "
+                "level is not derived. Skipping."
+            )
 
     def _dask_task_to_process_prefix_list(self, prefix_list: List[str]):
         """

tests/test_aind_bucket_indexer.py

@@ -1144,6 +1144,144 @@ def test_process_records(self, mock_dask_bag_map_parts: MagicMock):
         self.basic_job._process_records(example_records)
         mock_dask_bag_map_parts.assert_called()
 
+    @patch(
+        "aind_data_asset_indexer.aind_bucket_indexer."
+        "download_json_file_from_s3"
+    )
+    @patch("aind_data_asset_indexer.aind_bucket_indexer.does_s3_object_exist")
+    def test_get_data_level_for_prefix(
+        self,
+        mock_does_s3_object_exist: MagicMock,
+        mock_download_json_file_from_s3: MagicMock,
+    ):
+        """Tests _get_data_level_for_prefix method."""
+        mock_s3_client = MagicMock()
+        mock_does_s3_object_exist.return_value = True
+        mock_download_json_file_from_s3.return_value = self.example_md_record1[
+            "data_description"
+        ]
+
+        prefix = "ecephys_642478_2023-01-17_13-56-29"
+        data_level = self.basic_job._get_data_level_for_prefix(
+            s3_client=mock_s3_client,
+            bucket=self.basic_job.job_settings.s3_bucket,
+            prefix=prefix,
+        )
+        self.assertEqual("raw", data_level)
+        mock_does_s3_object_exist.assert_called_once_with(
+            s3_client=mock_s3_client,
+            bucket=self.basic_job.job_settings.s3_bucket,
+            key=f"{prefix}/data_description.json",
+        )
+        mock_download_json_file_from_s3.assert_called_once_with(
+            s3_client=mock_s3_client,
+            bucket=self.basic_job.job_settings.s3_bucket,
+            object_key=f"{prefix}/data_description.json",
+        )
+
+    @patch(
+        "aind_data_asset_indexer.aind_bucket_indexer."
+        "download_json_file_from_s3"
+    )
+    @patch("aind_data_asset_indexer.aind_bucket_indexer.does_s3_object_exist")
+    def test_get_data_level_for_prefix_no_file(
+        self,
+        mock_does_s3_object_exist: MagicMock,
+        mock_download_json_file_from_s3: MagicMock,
+    ):
+        """Tests _get_data_level_for_prefix method when there is no
+        data_description file."""
+        mock_s3_client = MagicMock()
+        mock_does_s3_object_exist.return_value = False
+
+        prefix = "ecephys_642478_2023-01-17_13-56-29"
+        data_level = self.basic_job._get_data_level_for_prefix(
+            s3_client=mock_s3_client,
+            bucket=self.basic_job.job_settings.s3_bucket,
+            prefix=prefix,
+        )
+        self.assertEqual(None, data_level)
+        mock_download_json_file_from_s3.assert_not_called()
+
+    @patch(
+        "aind_data_asset_indexer.aind_bucket_indexer."
+        "download_json_file_from_s3"
+    )
+    @patch("aind_data_asset_indexer.aind_bucket_indexer.does_s3_object_exist")
+    def test_get_data_level_for_prefix_invalid_file(
+        self,
+        mock_does_s3_object_exist: MagicMock,
+        mock_download_json_file_from_s3: MagicMock,
+    ):
+        """Tests _get_data_level_for_prefix method when the data_description
+        file is not valid json."""
+        mock_s3_client = MagicMock()
+        mock_does_s3_object_exist.return_value = True
+        mock_download_json_file_from_s3.return_value = None
+
+        prefix = "ecephys_642478_2023-01-17_13-56-29"
+        data_level = self.basic_job._get_data_level_for_prefix(
+            s3_client=mock_s3_client,
+            bucket=self.basic_job.job_settings.s3_bucket,
+            prefix=prefix,
+        )
+        self.assertEqual(None, data_level)
+
+    @patch(
+        "aind_data_asset_indexer.aind_bucket_indexer."
+        "upload_metadata_json_str_to_s3"
+    )
+    @patch(
+        "aind_data_asset_indexer.aind_bucket_indexer."
+        "cond_copy_then_sync_core_json_files"
+    )
+    @patch(
+        "aind_data_asset_indexer.aind_bucket_indexer."
+        "build_metadata_record_from_prefix"
+    )
+    @patch(
+        "aind_data_asset_indexer.aind_bucket_indexer.AindIndexBucketJob"
+        "._get_data_level_for_prefix"
+    )
+    @patch("aind_data_asset_indexer.aind_bucket_indexer.does_s3_object_exist")
+    @patch("aind_data_asset_indexer.aind_bucket_indexer.MetadataDbClient")
+    @patch("boto3.client")
+    def test_process_prefix_no_record_no_file_derived_no(
+        self,
+        mock_s3_client: MagicMock,
+        mock_docdb_client: MagicMock,
+        mock_does_s3_object_exist: MagicMock,
+        mock_get_data_level_for_prefix: MagicMock,
+        mock_build_metadata_record_from_prefix: MagicMock,
+        mock_cond_copy_then_sync_core_json_files: MagicMock,
+        mock_upload_metadata_json_str_to_s3: MagicMock,
+    ):
+        """Tests _process_prefix method when there is no record in DocDb,
+        there is no metadata.nd.json file in S3, and the
+        asset data level is not derived."""
+
+        mock_does_s3_object_exist.return_value = False
+        mock_get_data_level_for_prefix.return_value = "raw"
+        mock_build_metadata_record_from_prefix.return_value = None
+
+        location_to_id_map = dict()
+        with self.assertLogs(level="DEBUG") as captured:
+            self.basic_job._process_prefix(
+                s3_prefix="ecephys_642478_2023-01-17_13-56-29",
+                docdb_client=mock_docdb_client,
+                s3_client=mock_s3_client,
+                location_to_id_map=location_to_id_map,
+            )
+        expected_log_messages = [
+            "INFO:root:Metadata record for "
+            "s3://aind-ephys-data-dev-u5u0i5/"
+            "ecephys_642478_2023-01-17_13-56-29 not found in S3 and data "
+            "level is not derived. Skipping."
+        ]
+        self.assertEqual(expected_log_messages, captured.output)
+        mock_cond_copy_then_sync_core_json_files.assert_not_called()
+        mock_upload_metadata_json_str_to_s3.assert_not_called()
+
     @patch(
         "aind_data_asset_indexer.aind_bucket_indexer."
         "upload_metadata_json_str_to_s3"
@@ -1156,6 +1294,10 @@ def test_process_records(self, mock_dask_bag_map_parts: MagicMock):
         "aind_data_asset_indexer.aind_bucket_indexer."
         "build_metadata_record_from_prefix"
     )
+    @patch(
+        "aind_data_asset_indexer.aind_bucket_indexer.AindIndexBucketJob."
+        "_get_data_level_for_prefix"
+    )
     @patch("aind_data_asset_indexer.aind_bucket_indexer.does_s3_object_exist")
     @patch("aind_data_asset_indexer.aind_bucket_indexer.MetadataDbClient")
     @patch("boto3.client")
@@ -1164,6 +1306,7 @@ def test_process_prefix_no_record_no_file_build_no(
         mock_s3_client: MagicMock,
         mock_docdb_client: MagicMock,
         mock_does_s3_object_exist: MagicMock,
+        mock_get_data_level_for_prefix: MagicMock,
         mock_build_metadata_record_from_prefix: MagicMock,
         mock_cond_copy_then_sync_core_json_files: MagicMock,
         mock_upload_metadata_json_str_to_s3: MagicMock,
@@ -1173,6 +1316,7 @@ def test_process_prefix_no_record_no_file_build_no(
         build_metadata_record_from_prefix returns a None."""
 
         mock_does_s3_object_exist.return_value = False
+        mock_get_data_level_for_prefix.return_value = "derived"
         mock_build_metadata_record_from_prefix.return_value = None
 
         location_to_id_map = dict()
@@ -1204,6 +1348,10 @@ def test_process_prefix_no_record_no_file_build_no(
         "aind_data_asset_indexer.aind_bucket_indexer."
         "build_metadata_record_from_prefix"
     )
+    @patch(
+        "aind_data_asset_indexer.aind_bucket_indexer.AindIndexBucketJob."
+        "_get_data_level_for_prefix"
+    )
     @patch("aind_data_asset_indexer.aind_bucket_indexer.does_s3_object_exist")
     @patch("aind_data_asset_indexer.aind_bucket_indexer.MetadataDbClient")
     @patch("boto3.client")
@@ -1212,6 +1360,7 @@ def test_process_prefix_no_record_no_file_build_yes(
         mock_s3_client: MagicMock,
         mock_docdb_client: MagicMock,
         mock_does_s3_object_exist: MagicMock,
+        mock_get_data_level_for_prefix: MagicMock,
         mock_build_metadata_record_from_prefix: MagicMock,
         mock_cond_copy_then_sync_core_json_files: MagicMock,
         mock_upload_metadata_json_str_to_s3: MagicMock,
@@ -1222,6 +1371,7 @@ def test_process_prefix_no_record_no_file_build_yes(
 
         expected_prefix = "ecephys_642478_2023-01-17_13-56-29"
         mock_does_s3_object_exist.return_value = False
+        mock_get_data_level_for_prefix.return_value = "derived"
         mock_build_metadata_record_from_prefix.return_value = json.dumps(
             self.example_md_record
         )