Extending data_manager (#2196)

* added API call to query metadata entries

* exposing to data_mater

* added tests

* fixing check

* added more debugging

* this should really be working

* removing pointless statement

* figuring out types

* checking before uploading

* using correct object

* no need to assert this

* assert it dose not exsit

* Update packages/simcore-sdk/src/simcore_sdk/node_data/data_manager.py

Co-authored-by: Sylvain <[email protected]>

* applied renaming

Co-authored-by: Andrei Neagu <[email protected]>
Co-authored-by: Sylvain <[email protected]>

Author: 3 people

packages/simcore-sdk/src/simcore_sdk/node_data/data_manager.py

@@ -69,3 +69,13 @@ async def pull(file_or_folder: Path):
             archive_to_extract=str(archive_file), destination_folder=file_or_folder
         )
         log.info("extraction completed")
+
+
+async def is_file_present_in_storage(file_path: Path) -> bool:
+    """
+    :retruns True if an entry is present inside the files_metadata else False
+    """
+    return await filemanager.entry_exists(
+        store_id=0,  # this is for simcore.s3
+        s3_object=_create_s3_object(file_path),
+    )

packages/simcore-sdk/src/simcore_sdk/node_ports/filemanager.py

@@ -297,3 +297,21 @@ async def upload_file(
                 return store_id, e_tag
 
     raise exceptions.S3InvalidPathError(s3_object)
+
+
+async def entry_exists(store_id: str, s3_object: str) -> bool:
+    """Returns True if metadata for s3_object is present"""
+    user_id = config.USER_ID
+    with api_client() as client:
+        api = UsersApi(client)
+        try:
+            result = await api.get_file_metadata(s3_object, store_id, user_id)
+            is_metadata_present = result.data.object_name == s3_object
+            return is_metadata_present
+        except Exception:  # pylint: disable=broad-except
+            log.warning(
+                "There is no metadata for requested store_id=%s s3_object=%s",
+                store_id,
+                s3_object,
+            )
+            return False

packages/simcore-sdk/tests/integration/test_filemanager.py

@@ -126,3 +126,48 @@ async def test_invalid_store(
         await filemanager.download_file_from_s3(
             store_name=store, s3_object=file_id, local_folder=download_folder
         )
+
+
+async def test_valid_metadata(
+    tmpdir: Path,
+    bucket: str,
+    filemanager_cfg: None,
+    user_id: str,
+    file_uuid: str,
+    s3_simcore_location: str,
+):
+    file_path = Path(tmpdir) / "test.test"
+    file_path.write_text("I am a test file")
+    assert file_path.exists()
+
+    file_id = file_uuid(file_path)
+    store_id, e_tag = await filemanager.upload_file(
+        store_id=s3_simcore_location, s3_object=file_id, local_file_path=file_path
+    )
+    assert store_id == s3_simcore_location
+    assert e_tag
+
+    is_metadata_present = await filemanager.entry_exists(
+        store_id=store_id, s3_object=file_id
+    )
+
+    assert is_metadata_present is True
+
+
+async def test_invalid_metadata(
+    tmpdir: Path,
+    bucket: str,
+    filemanager_cfg: None,
+    user_id: str,
+    file_uuid: str,
+    s3_simcore_location: str,
+):
+    file_path = Path(tmpdir) / "test.test"
+    file_id = file_uuid(file_path)
+    assert file_path.exists() is False
+
+    is_metadata_present = await filemanager.entry_exists(
+        store_id=s3_simcore_location, s3_object=file_id
+    )
+
+    assert is_metadata_present is False