Skip to content

Commit e80861a

Browse files
jtyoung84jtyoung84
authored
Merge pull request #96 from AllenNeuralDynamics/release-v0.11.0
Release v0.11.0
2 parents 7342be3 + 352e162 commit e80861a

File tree

10 files changed

+37
-32
lines changed

10 files changed

+37
-32
lines changed

‎pyproject.toml‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -20,10 +20,10 @@ dependencies = [
2020
"boto3",
2121
"boto3-stubs[s3]",
2222
"pydantic-settings>=2.0",
23-
"pydantic>=2.0,<2.7",
23+
"pydantic>=2.7,<2.9",
2424
"pymongo==4.3.3",
2525
"dask==2023.5.0",
26-
"aind-data-schema==0.33.3",
26+
"aind-data-schema==1.0.0",
2727
"aind-codeocean-api==0.5.0",
2828
]
2929

‎src/aind_data_asset_indexer/__init__.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,3 @@
11
"""Package"""
22

3-
__version__ = "0.10.0"
3+
__version__ = "0.11.0"

‎src/aind_data_asset_indexer/codeocean_bucket_indexer.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -248,7 +248,7 @@ def _process_codeocean_record(
248248
"""
249249
location = codeocean_record["location"]
250250
created = codeocean_record["created"]
251-
external_links = [codeocean_record["external_links"]]
251+
external_links = codeocean_record["external_links"]
252252
name = codeocean_record["name"]
253253
url_parts = get_s3_bucket_and_prefix(location)
254254
bucket = url_parts["bucket"]

‎src/aind_data_asset_indexer/utils.py‎

Lines changed: 7 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,7 @@
1111

1212
from aind_codeocean_api.codeocean import CodeOceanClient
1313
from aind_data_schema.core.data_description import DataLevel, DataRegex
14-
from aind_data_schema.core.metadata import Metadata
14+
from aind_data_schema.core.metadata import ExternalPlatforms, Metadata
1515
from aind_data_schema.utils.json_writer import SchemaWriter
1616
from botocore.exceptions import ClientError
1717
from mypy_boto3_s3 import S3Client
@@ -567,7 +567,7 @@ def build_metadata_record_from_prefix(
567567
s3_client: S3Client,
568568
optional_name: Optional[str] = None,
569569
optional_created: Optional[datetime] = None,
570-
optional_external_links: Optional[List[dict]] = None,
570+
optional_external_links: Optional[Dict[str, List[str]]] = None,
571571
) -> Optional[str]:
572572
"""
573573
For a given bucket and prefix, this method will return a JSON string
@@ -585,7 +585,7 @@ def build_metadata_record_from_prefix(
585585
s3_prefix. Default is None.
586586
optional_created: Optional[datetime]
587587
User can override created datetime. Default is None.
588-
optional_external_links: Optional[List[dict]]
588+
optional_external_links: Optional[Dict[str, List[str]]]
589589
User can provide external_links. Default is None.
590590
591591
Returns
@@ -1048,7 +1048,7 @@ def get_all_processed_codeocean_asset_records(
10481048
{"name": data_asset_name,
10491049
"location": data_asset_location,
10501050
"created": data_asset_created,
1051-
"external_links": {"Code Ocean": data_asset_id}
1051+
"external_links": {"Code Ocean": [data_asset_id]}
10521052
}
10531053
}
10541054
@@ -1091,7 +1091,9 @@ def get_all_processed_codeocean_asset_records(
10911091
"name": data_asset_name,
10921092
"location": location,
10931093
"created": created_datetime,
1094-
"external_links": {"Code Ocean": data_asset_id},
1094+
"external_links": {
1095+
ExternalPlatforms.CODEOCEAN.value: [data_asset_id]
1096+
},
10951097
}
10961098
# Occasionally, there are duplicate items returned. This is one
10971099
# way to remove the duplicates.

‎tests/resources/utils/example_metadata.nd.json‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -87,8 +87,9 @@
8787
"pipeline_version": null,
8888
"schema_version": "0.1.0"
8989
},
90+
"quality_control": null,
9091
"rig": null,
91-
"schema_version": "0.2.7",
92+
"schema_version": "1.0.0",
9293
"session": null,
9394
"subject": {
9495
"background_strain": null,

‎tests/resources/utils/example_metadata1.nd.json‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -95,8 +95,9 @@
9595
"pipeline_version": null,
9696
"schema_version": "0.2.5"
9797
},
98+
"quality_control": null,
9899
"rig": null,
99-
"schema_version": "0.2.7",
100+
"schema_version": "1.0.0",
100101
"session": null,
101102
"subject": {
102103
"background_strain": null,

‎tests/resources/utils/example_metadata2.nd.json‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -173,8 +173,9 @@
173173
},
174174
"schema_version": "0.3.1"
175175
},
176+
"quality_control": null,
176177
"rig": null,
177-
"schema_version": "0.2.7",
178+
"schema_version": "1.0.0",
178179
"session": null,
179180
"subject": {
180181
"background_strain": null,

‎tests/test_aind_bucket_indexer.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -73,7 +73,7 @@ def test_write_root_file_with_record_info_same_hash(
7373
"last_modified": datetime(
7474
2024, 5, 15, 17, 41, 28, tzinfo=timezone.utc
7575
),
76-
"e_tag": '"e6dd2b7ab819f7a0fc21dba512a4071b"',
76+
"e_tag": '"275d922d2a1e547f2e0f35b5cc54f493"',
7777
"version_id": "version_id",
7878
},
7979
prefix="ecephys_642478_2023-01-17_13-56-29",

‎tests/test_codeocean_bucket_indexer.py‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -54,7 +54,7 @@ def setUpClass(cls) -> None:
5454
2024, 6, 12, 21, 21, 28, tzinfo=timezone.utc
5555
),
5656
"external_links": {
57-
"Code Ocean": "11ee1e1e-11e1-1111-1111-e11eeeee1e11"
57+
"Code Ocean": ["11ee1e1e-11e1-1111-1111-e11eeeee1e11"]
5858
},
5959
},
6060
{
@@ -69,7 +69,7 @@ def setUpClass(cls) -> None:
6969
2024, 6, 12, 19, 45, 59, tzinfo=timezone.utc
7070
),
7171
"external_links": {
72-
"Code Ocean": "666666cc-66cc-6c66-666c-6c66c6666666"
72+
"Code Ocean": ["666666cc-66cc-6c66-666c-6c66c6666666"]
7373
},
7474
},
7575
]
@@ -79,10 +79,10 @@ def setUpClass(cls) -> None:
7979
"ecephys_642478_2023-01-17_13-56-29/instrument.json": None,
8080
"ecephys_642478_2023-01-17_13-56-29/procedures.json": None,
8181
"ecephys_642478_2023-01-17_13-56-29/processing.json": None,
82+
"ecephys_642478_2023-01-17_13-56-29/quality_control.json": None,
8283
"ecephys_642478_2023-01-17_13-56-29/rig.json": None,
8384
"ecephys_642478_2023-01-17_13-56-29/session.json": None,
8485
"ecephys_642478_2023-01-17_13-56-29/subject.json": None,
85-
"ecephys_642478_2023-01-17_13-56-29/mri_session.json": None,
8686
}
8787
cls.example_docdb_records = [
8888
{

‎tests/test_utils.py‎

Lines changed: 16 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -64,10 +64,10 @@ def load_json_file(filename: str) -> dict:
6464
"instrument",
6565
"procedures",
6666
"processing",
67+
"quality_control",
6768
"rig",
6869
"session",
6970
"subject",
70-
"mri_session",
7171
]
7272
cls.example_core_files = example_core_files
7373
example_pages = load_json_file("example_pages_response.json")
@@ -187,7 +187,7 @@ def test__log_message_false(self, mock_log: MagicMock):
187187
def test_compute_md5_hash(self):
188188
"""Tests compute_md5_hash method"""
189189
md5_hash = compute_md5_hash(json.dumps(self.example_metadata_nd))
190-
self.assertEqual("e6dd2b7ab819f7a0fc21dba512a4071b", md5_hash)
190+
self.assertEqual("275d922d2a1e547f2e0f35b5cc54f493", md5_hash)
191191

192192
def test_is_dict_corrupt(self):
193193
"""Tests is_dict_corrupt method"""
@@ -746,6 +746,7 @@ def test_build_metadata_record_from_prefix(
746746
"e_tag": '"f4827f025e79bafeb6947e14c4e3b51a"',
747747
"version_id": "jWWT0Xrb8_nE9t5C.nTlLElpYJoURbv_",
748748
},
749+
"ecephys_642478_2023-01-17_13-56-29/quality_control.json": None,
749750
"ecephys_642478_2023-01-17_13-56-29/rig.json": None,
750751
"ecephys_642478_2023-01-17_13-56-29/session.json": None,
751752
"ecephys_642478_2023-01-17_13-56-29/subject.json": {
@@ -755,7 +756,6 @@ def test_build_metadata_record_from_prefix(
755756
"e_tag": '"92734946c64fc87408ef79e5e92937bc"',
756757
"version_id": "XS0p7m6wWNTHG_F3P76D7AUXtE23BakR",
757758
},
758-
"ecephys_642478_2023-01-17_13-56-29/mri_session.json": None,
759759
}
760760
mock_download_json_file.side_effect = [
761761
self.example_processing,
@@ -811,10 +811,10 @@ def test_build_metadata_record_from_prefix_with_optional_fields(
811811
"ecephys_642478_2023-01-17_13-56-29/instrument.json": None,
812812
"ecephys_642478_2023-01-17_13-56-29/procedures.json": None,
813813
"ecephys_642478_2023-01-17_13-56-29/processing.json": None,
814+
"ecephys_642478_2023-01-17_13-56-29/quality_control.json": None,
814815
"ecephys_642478_2023-01-17_13-56-29/rig.json": None,
815816
"ecephys_642478_2023-01-17_13-56-29/session.json": None,
816817
"ecephys_642478_2023-01-17_13-56-29/subject.json": None,
817-
"ecephys_642478_2023-01-17_13-56-29/mri_session.json": None,
818818
}
819819
# noinspection PyTypeChecker
820820
md = json.loads(
@@ -824,15 +824,15 @@ def test_build_metadata_record_from_prefix_with_optional_fields(
824824
s3_client=mock_s3_client,
825825
optional_name="ecephys_642478_2023-01-17_13-56-29",
826826
optional_created=datetime(2020, 1, 2, 3, 4, 5),
827-
optional_external_links=[{"Code Ocean": "123-456"}],
827+
optional_external_links={"Code Ocean": ["123-456"]},
828828
)
829829
)
830830
mock_get_dict_of_file_info.assert_called_once()
831831
mock_download_json_file.assert_not_called()
832832
self.assertEqual("s3://code-ocean-bucket/abc-123", md["location"])
833833
self.assertEqual("ecephys_642478_2023-01-17_13-56-29", md["name"])
834834
self.assertEqual("2020-01-02T03:04:05", md["created"])
835-
self.assertEqual([{"Code Ocean": "123-456"}], md["external_links"])
835+
self.assertEqual({"Code Ocean": ["123-456"]}, md["external_links"])
836836

837837
@patch("aind_data_asset_indexer.utils.Metadata.model_construct")
838838
@patch("boto3.client")
@@ -859,6 +859,7 @@ def test_build_metadata_record_from_prefix_error(
859859
"e_tag": '"f4827f025e79bafeb6947e14c4e3b51a"',
860860
"version_id": "jWWT0Xrb8_nE9t5C.nTlLElpYJoURbv_",
861861
},
862+
"ecephys_642478_2023-01-17_13-56-29/quality_control.json": None,
862863
"ecephys_642478_2023-01-17_13-56-29/rig.json": None,
863864
"ecephys_642478_2023-01-17_13-56-29/session.json": None,
864865
"ecephys_642478_2023-01-17_13-56-29/subject.json": {
@@ -868,7 +869,6 @@ def test_build_metadata_record_from_prefix_error(
868869
"e_tag": '"92734946c64fc87408ef79e5e92937bc"',
869870
"version_id": "XS0p7m6wWNTHG_F3P76D7AUXtE23BakR",
870871
},
871-
"ecephys_642478_2023-01-17_13-56-29/mri_session.json": None,
872872
}
873873
mock_download_json_file.side_effect = [
874874
self.example_processing,
@@ -949,6 +949,7 @@ def test_sync_core_json_files(
949949
"e_tag": f'"{md5_hash_processing_unchanged}"',
950950
"version_id": "jWWT0Xrb8_nE9t5C.nTlLElpYJoURbv_",
951951
},
952+
f"{pfx}/quality_control.json": None,
952953
f"{pfx}/rig.json": {
953954
"last_modified": datetime(
954955
2023, 11, 4, 1, 13, 41, tzinfo=timezone.utc
@@ -964,7 +965,6 @@ def test_sync_core_json_files(
964965
"e_tag": f'"{md5_hash_subject_unchanged}"',
965966
"version_id": "XS0p7m6wWNTHG_F3P76D7AUXtE23BakR",
966967
},
967-
f"{pfx}/mri_session.json": None,
968968
}
969969
mock_upload_core_record.return_value = "mock_upload_response"
970970
mock_s3_client.delete_object.return_value = "mock_delete_response"
@@ -1020,6 +1020,10 @@ def test_sync_core_json_files(
10201020
f"processing is up-to-date in {s3_loc}/processing.json. "
10211021
"Skipping."
10221022
),
1023+
(
1024+
f"quality_control not found in metadata.nd.json for {pfx} nor "
1025+
f"in {s3_loc}/quality_control.json! Skipping."
1026+
),
10231027
(
10241028
f"rig not found in metadata.nd.json for {pfx} but {s3_loc}/"
10251029
"rig.json exists! Deleting."
@@ -1030,10 +1034,6 @@ def test_sync_core_json_files(
10301034
f"{s3_loc}/session.json! Skipping."
10311035
),
10321036
f"subject is up-to-date in {s3_loc}/subject.json. Skipping.",
1033-
(
1034-
f"mri_session not found in metadata.nd.json for {pfx} nor in "
1035-
f"{s3_loc}/mri_session.json! Skipping."
1036-
),
10371037
]
10381038
actual_log_messages = [
10391039
c[1]["message"] for c in mock_log_message.call_args_list
@@ -1072,6 +1072,7 @@ def test_cond_copy_then_sync_core_json_files(
10721072
"e_tag": '"7ebb961de9e9b00accfd1358e4561ec1"',
10731073
"version_id": "jWWT0Xrb8_nE9t5C.nTlLElpYJoURbv_",
10741074
},
1075+
f"{pfx}/quality_control.json": None,
10751076
f"{pfx}/rig.json": None,
10761077
f"{pfx}/session.json": None,
10771078
f"{pfx}/subject.json": {
@@ -1081,7 +1082,6 @@ def test_cond_copy_then_sync_core_json_files(
10811082
"e_tag": '"8b8cd50a6cf1f3f667be98a69db2ad89"',
10821083
"version_id": "XS0p7m6wWNTHG_F3P76D7AUXtE23BakR",
10831084
},
1084-
f"{pfx}/mri_session.json": None,
10851085
}
10861086
cond_copy_then_sync_core_json_files(
10871087
metadata_json=json.dumps(self.example_metadata_nd),
@@ -1164,6 +1164,7 @@ def test_cond_copy_then_sync_core_json_files_mismatch(
11641164
"e_tag": '"7ebb961de9e9b00accfd1358e4561ec1"',
11651165
"version_id": "jWWT0Xrb8_nE9t5C.nTlLElpYJoURbv_",
11661166
},
1167+
f"{pfx}/quality_control.json": None,
11671168
f"{pfx}/rig.json": {
11681169
"last_modified": datetime(
11691170
2022, 5, 5, 1, 13, 41, tzinfo=timezone.utc
@@ -1179,7 +1180,6 @@ def test_cond_copy_then_sync_core_json_files_mismatch(
11791180
"e_tag": '"8b8cd50a6cf1f3f667be98a69db2ad89"',
11801181
"version_id": "XS0p7m6wWNTHG_F3P76D7AUXtE23BakR",
11811182
},
1182-
f"{pfx}/mri_session.json": None,
11831183
}
11841184

11851185
cond_copy_then_sync_core_json_files(
@@ -1436,7 +1436,7 @@ def test_get_all_processed_codeocean_asset_records(
14361436
2024, 6, 12, 21, 21, 28, tzinfo=timezone.utc
14371437
),
14381438
"external_links": {
1439-
"Code Ocean": "11ee1e1e-11e1-1111-1111-e11eeeee1e11"
1439+
"Code Ocean": ["11ee1e1e-11e1-1111-1111-e11eeeee1e11"]
14401440
},
14411441
},
14421442
"s3://some_co_bucket/666666cc-66cc-6c66-666c-6c66c6666666": {
@@ -1451,7 +1451,7 @@ def test_get_all_processed_codeocean_asset_records(
14511451
2024, 6, 12, 19, 45, 59, tzinfo=timezone.utc
14521452
),
14531453
"external_links": {
1454-
"Code Ocean": "666666cc-66cc-6c66-666c-6c66c6666666"
1454+
"Code Ocean": ["666666cc-66cc-6c66-666c-6c66c6666666"]
14551455
},
14561456
},
14571457
}

0 commit comments

Comments
 (0)