everywhere TypeAdapter

Author: sanderegg

services/storage/src/simcore_service_storage/datcore_adapter/datcore_adapter.py

@@ -9,7 +9,7 @@
 from aiohttp.client import ClientSession
 from models_library.api_schemas_storage import DatCoreDatasetName
 from models_library.users import UserID
-from pydantic import AnyUrl, parse_obj_as
+from pydantic import AnyUrl, TypeAdapter
 from servicelib.aiohttp.application_keys import APP_CONFIG_KEY
 from servicelib.aiohttp.client_session import get_client_session
 from servicelib.utils import logged_gather
@@ -229,7 +229,7 @@ async def get_file_download_presigned_link(
         dict[str, Any],
         await _request(app, api_key, api_secret, "GET", f"/files/{file_id}"),
     )
-    url: AnyUrl = parse_obj_as(AnyUrl, file_download_data["link"])
+    url: AnyUrl = TypeAdapter(AnyUrl).validate_python(file_download_data["link"])
     return url
 
 

services/storage/src/simcore_service_storage/s3_utils.py

@@ -2,7 +2,7 @@
 from collections import defaultdict
 from dataclasses import dataclass, field
 
-from pydantic import ByteSize, parse_obj_as
+from pydantic import ByteSize, TypeAdapter
 from servicelib.aiohttp.long_running_tasks.server import (
     ProgressMessage,
     ProgressPercent,
@@ -55,7 +55,7 @@ def finalize_transfer(self) -> None:
     def copy_transfer_cb(self, total_bytes_copied: int, *, file_name: str) -> None:
         _logger.debug(
             "Copied %s of %s",
-            parse_obj_as(ByteSize, total_bytes_copied).human_readable(),
+            TypeAdapter(ByteSize).validate_python(total_bytes_copied).human_readable(),
             file_name,
         )
         self._file_total_bytes_copied[file_name] = total_bytes_copied
@@ -66,7 +66,7 @@ def copy_transfer_cb(self, total_bytes_copied: int, *, file_name: str) -> None:
     def upload_transfer_cb(self, bytes_transferred: int, *, file_name: str) -> None:
         _logger.debug(
             "Uploaded %s of %s",
-            parse_obj_as(ByteSize, bytes_transferred).human_readable(),
+            TypeAdapter(ByteSize).validate_python(bytes_transferred).human_readable(),
             file_name,
         )
         self._file_total_bytes_copied[file_name] += bytes_transferred

services/storage/src/simcore_service_storage/simcore_s3_dsm.py

@@ -35,7 +35,7 @@
     StorageFileID,
 )
 from models_library.users import UserID
-from pydantic import AnyUrl, ByteSize, NonNegativeInt, TypeAdapter, parse_obj_as
+from pydantic import AnyUrl, ByteSize, NonNegativeInt, TypeAdapter
 from servicelib.aiohttp.client_session import get_client_session
 from servicelib.aiohttp.long_running_tasks.server import TaskProgress
 from servicelib.logging_utils import log_context
@@ -257,7 +257,7 @@ async def get_file(self, user_id: UserID, file_id: StorageFileID) -> FileMetaDat
                 raise FileAccessRightError(access_right="read", file_id=file_id)
 
             fmd = await db_file_meta_data.get(
-                conn, parse_obj_as(SimcoreS3FileID, file_id)
+                conn, TypeAdapter(SimcoreS3FileID).validate_python(file_id)
             )
         if is_file_entry_valid(fmd):
             return convert_db_to_model(fmd)
@@ -412,7 +412,7 @@ async def complete_file_upload(
             if not can.write:
                 raise FileAccessRightError(access_right="write", file_id=file_id)
             fmd = await db_file_meta_data.get(
-                conn, parse_obj_as(SimcoreS3FileID, file_id)
+                conn, TypeAdapter(SimcoreS3FileID).validate_python(file_id)
             )
 
         if is_valid_managed_multipart_upload(fmd.upload_id):
@@ -460,12 +460,12 @@ async def create_file_download_link(
             ):
                 raise S3KeyNotFoundError(key=file_id, bucket=self.simcore_bucket_name)
             return await self.__get_link(
-                parse_obj_as(SimcoreS3FileID, file_id), link_type
+                TypeAdapter(SimcoreS3FileID).validate_python(file_id), link_type
             )
         # standard file link
         async with self.engine.acquire() as conn:
             fmd = await db_file_meta_data.get(
-                conn, parse_obj_as(SimcoreS3FileID, file_id)
+                conn, TypeAdapter(SimcoreS3FileID).validate_python(file_id)
             )
         if not is_file_entry_valid(fmd):
             # try lazy update
@@ -487,9 +487,8 @@ async def __ensure_read_access_rights(
     async def __get_link(
         self, s3_file_id: SimcoreS3FileID, link_type: LinkType
     ) -> AnyUrl:
-        link: AnyUrl = parse_obj_as(
-            AnyUrl,
-            f"s3://{self.simcore_bucket_name}/{urllib.parse.quote(s3_file_id)}",
+        link: AnyUrl = TypeAdapter(AnyUrl).validate_python(
+            f"s3://{self.simcore_bucket_name}/{urllib.parse.quote(s3_file_id)}"
         )
         if link_type == LinkType.PRESIGNED:
             link = await get_s3_client(self.app).create_single_presigned_download_link(
@@ -528,7 +527,7 @@ async def delete_file(
             # NOTE: deleting might be slow, so better ensure we release the connection
             async with self.engine.acquire() as conn:
                 file: FileMetaDataAtDB = await db_file_meta_data.get(
-                    conn, parse_obj_as(SimcoreS3FileID, file_id)
+                    conn, TypeAdapter(SimcoreS3FileID).validate_python(file_id)
                 )
             await get_s3_client(self.app).delete_objects_recursively(
                 bucket=file.bucket_name,
@@ -632,9 +631,9 @@ async def deep_copy_project_simcore_s3(
                     limit=_MAX_PARALLEL_S3_CALLS,
                 )
             total_num_of_files = sum(n for _, n in sizes_and_num_files)
-            src_project_total_data_size: ByteSize = parse_obj_as(
-                ByteSize, sum(n for n, _ in sizes_and_num_files)
-            )
+            src_project_total_data_size: ByteSize = TypeAdapter(
+                ByteSize
+            ).validate_python(sum(n for n, _ in sizes_and_num_files))
         with log_context(
             _logger,
             logging.INFO,
@@ -660,7 +659,7 @@ async def deep_copy_project_simcore_s3(
                         self._copy_path_s3_s3(
                             user_id,
                             src_fmd=src_fmd,
-                            dst_file_id=SimcoreS3FileID(
+                            dst_file_id=TypeAdapter(SimcoreS3FileID).validate_python(
                                 f"{dst_project_uuid}/{new_node_id}/{src_fmd.object_name.split('/', maxsplit=2)[-1]}"
                             ),
                             bytes_transfered_cb=s3_transfered_data_cb.copy_transfer_cb,
@@ -718,7 +717,7 @@ async def _get_size_and_num_files(
             total_size += sum(x.size for x in s3_objects)
             total_num_s3_objects += len(s3_objects)
 
-        return parse_obj_as(ByteSize, total_size), total_num_s3_objects
+        return TypeAdapter(ByteSize).validate_python(total_size), total_num_s3_objects
 
     async def search_owned_files(
         self,
@@ -759,7 +758,7 @@ async def create_soft_link(
     ) -> FileMetaData:
         async with self.engine.acquire() as conn:
             if await db_file_meta_data.exists(
-                conn, parse_obj_as(SimcoreS3FileID, link_file_id)
+                conn, TypeAdapter(SimcoreS3FileID).validate_python(link_file_id)
             ):
                 raise LinkAlreadyExistsError(file_id=link_file_id)
         # validate target_uuid
@@ -908,7 +907,7 @@ async def _update_fmd_from_other(
             s3_metadata = await get_s3_client(self.app).get_object_metadata(
                 bucket=fmd.bucket_name, object_key=fmd.object_name
             )
-            fmd.file_size = parse_obj_as(ByteSize, s3_metadata.size)
+            fmd.file_size = TypeAdapter(ByteSize).validate_python(s3_metadata.size)
             fmd.last_modified = s3_metadata.last_modified
             fmd.entity_tag = s3_metadata.e_tag
         else:
@@ -945,12 +944,12 @@ async def _update_database_from_storage(
         s3_metadata = await self._get_s3_metadata(fmd)
         if not fmd.is_directory:
             assert isinstance(s3_metadata, S3MetaData)  # nosec
-            fmd.file_size = parse_obj_as(ByteSize, s3_metadata.size)
+            fmd.file_size = TypeAdapter(ByteSize).validate_python(s3_metadata.size)
             fmd.last_modified = s3_metadata.last_modified
             fmd.entity_tag = s3_metadata.e_tag
         elif fmd.is_directory:
             assert isinstance(s3_metadata, S3DirectoryMetaData)  # nosec
-            fmd.file_size = parse_obj_as(ByteSize, s3_metadata.size)
+            fmd.file_size = TypeAdapter(ByteSize).validate_python(s3_metadata.size)
         fmd.upload_expires_at = None
         fmd.upload_id = None
         async with self.engine.acquire() as conn:
@@ -978,7 +977,9 @@ async def _copy_file_datcore_s3(
         )
         assert dc_link.path  # nosec
         filename = Path(dc_link.path).name
-        dst_file_id = SimcoreS3FileID(f"{dest_project_id}/{dest_node_id}/{filename}")
+        dst_file_id = TypeAdapter(SimcoreS3FileID).validate_python(
+            f"{dest_project_id}/{dest_node_id}/{filename}"
+        )
         _logger.debug("copying %s to %s", f"{source_uuid=}", f"{dst_file_id=}")
 
         with tempfile.TemporaryDirectory() as tmpdir:
@@ -1075,7 +1076,7 @@ async def _create_fmd_for_upload(
         )
         fmd = FileMetaData.from_simcore_node(
             user_id=user_id,
-            file_id=parse_obj_as(SimcoreS3FileID, file_id),
+            file_id=TypeAdapter(SimcoreS3FileID).validate_python(file_id),
             bucket=self.simcore_bucket_name,
             location_id=self.location_id,
             location_name=self.location_name,
@@ -1092,7 +1093,9 @@ def create_simcore_s3_data_manager(app: web.Application) -> SimcoreS3DataManager
     assert cfg.STORAGE_S3  # nosec
     return SimcoreS3DataManager(
         engine=app[APP_AIOPG_ENGINE_KEY],
-        simcore_bucket_name=parse_obj_as(S3BucketName, cfg.STORAGE_S3.S3_BUCKET_NAME),
+        simcore_bucket_name=TypeAdapter(S3BucketName).validate_python(
+            cfg.STORAGE_S3.S3_BUCKET_NAME
+        ),
         app=app,
         settings=cfg,
     )

services/storage/tests/conftest.py

@@ -39,7 +39,7 @@
 from models_library.projects_nodes_io import LocationID, SimcoreS3FileID
 from models_library.users import UserID
 from models_library.utils.fastapi_encoders import jsonable_encoder
-from pydantic import ByteSize, parse_obj_as
+from pydantic import ByteSize, TypeAdapter
 from pytest_mock import MockerFixture
 from pytest_simcore.helpers.assert_checks import assert_status
 from pytest_simcore.helpers.logging_tools import log_context
@@ -259,7 +259,7 @@ async def _getter(file_id: SimcoreS3FileID) -> FileMetaDataGet:
         data, error = await assert_status(response, status.HTTP_200_OK)
         assert not error
         assert data
-        received_fmd = parse_obj_as(FileMetaDataGet, data)
+        received_fmd = TypeAdapter(FileMetaDataGet).validate_python(data)
         assert received_fmd
         return received_fmd
 
@@ -291,7 +291,7 @@ async def _link_creator(
         data, error = await assert_status(response, status.HTTP_200_OK)
         assert not error
         assert data
-        received_file_upload = parse_obj_as(FileUploadSchema, data)
+        received_file_upload = TypeAdapter(FileUploadSchema).validate_python(data)
         assert received_file_upload
         file_params.append((user_id, location_id, file_id))
         return received_file_upload
@@ -430,7 +430,7 @@ def _creator(
         if file_base_path:
             s3_file_name = f"{file_base_path / file_name}"
         clean_path = Path(f"{project_id}/{node_id}/{s3_file_name}")
-        return SimcoreS3FileID(f"{clean_path}")
+        return TypeAdapter(SimcoreS3FileID).validate_python(f"{clean_path}")
 
     return _creator
 
@@ -470,7 +470,7 @@ async def _directory_creator(dir_name: str):
         assert len(directory_file_upload.urls) == 1
 
         # complete the upload
-        complete_url = URL(directory_file_upload.links.complete_upload).relative()
+        complete_url = URL(f"{directory_file_upload.links.complete_upload}").relative()
         response = await client.post(
             f"{complete_url}",
             json=jsonable_encoder(FileUploadCompletionBody(parts=[])),
@@ -480,7 +480,7 @@ async def _directory_creator(dir_name: str):
         assert not error
         assert data
         file_upload_complete_response = FileUploadCompleteResponse.parse_obj(data)
-        state_url = URL(file_upload_complete_response.links.state).relative()
+        state_url = URL(f"{file_upload_complete_response.links.state}").relative()
 
         # check that it finished updating
         assert client.app
@@ -535,7 +535,9 @@ async def _create_file(s: int, f: int):
             await storage_s3_client.upload_file(
                 bucket=storage_s3_bucket,
                 file=file,
-                object_key=SimcoreS3FileID(f"{clean_path}"),
+                object_key=TypeAdapter(SimcoreS3FileID).validate_python(
+                    f"{clean_path}"
+                ),
                 bytes_transfered_cb=None,
             )
 

services/storage/tests/fixtures/data_models.py

@@ -18,7 +18,7 @@
 from models_library.projects import ProjectID
 from models_library.projects_nodes_io import NodeID, SimcoreS3FileID
 from models_library.users import UserID
-from pydantic import ByteSize, parse_obj_as
+from pydantic import ByteSize, TypeAdapter
 from pytest_simcore.helpers.faker_factories import random_project, random_user
 from servicelib.utils import limited_gather
 from simcore_postgres_database.models.project_to_groups import project_to_groups
@@ -45,7 +45,7 @@ async def _user_context(aiopg_engine: Engine, *, name: str) -> AsyncIterator[Use
     assert isinstance(row.id, int)
 
     try:
-        yield UserID(row.id)
+        yield TypeAdapter(UserID).validate_python(row.id)
     finally:
         async with aiopg_engine.acquire() as conn:
             await conn.execute(users.delete().where(users.c.id == row.id))
@@ -149,7 +149,7 @@ async def project_id(
 async def collaborator_id(aiopg_engine: Engine) -> AsyncIterator[UserID]:
 
     async with _user_context(aiopg_engine, name="collaborator") as new_user_id:
-        yield UserID(new_user_id)
+        yield TypeAdapter(UserID).validate_python(new_user_id)
 
 
 @pytest.fixture
@@ -177,7 +177,7 @@ async def _() -> None:
             )
             row = await result.fetchone()
             assert row
-            access_rights: dict[str, Any] = row[projects.c.access_rights]
+            access_rights: dict[str | int, Any] = row[projects.c.access_rights]
 
             access_rights[await _get_user_group(conn, user_id)] = {
                 "read": True,
@@ -279,22 +279,19 @@ async def random_project_with_files(
     async def _creator(
         num_nodes: int = 12,
         file_sizes: tuple[ByteSize, ...] = (
-            parse_obj_as(ByteSize, "7Mib"),
-            parse_obj_as(ByteSize, "110Mib"),
-            parse_obj_as(ByteSize, "1Mib"),
+            TypeAdapter(ByteSize).validate_python("7Mib"),
+            TypeAdapter(ByteSize).validate_python("110Mib"),
+            TypeAdapter(ByteSize).validate_python("1Mib"),
         ),
         file_checksums: tuple[SHA256Str, ...] = (
-            parse_obj_as(
-                SHA256Str,
-                "311e2e130d83cfea9c3b7560699c221b0b7f9e5d58b02870bd52b695d8b4aabd",
+            TypeAdapter(SHA256Str).validate_python(
+                "311e2e130d83cfea9c3b7560699c221b0b7f9e5d58b02870bd52b695d8b4aabd"
             ),
-            parse_obj_as(
-                SHA256Str,
-                "08e297db979d3c84f6b072c2a1e269e8aa04e82714ca7b295933a0c9c0f62b2e",
+            TypeAdapter(SHA256Str).validate_python(
+                "08e297db979d3c84f6b072c2a1e269e8aa04e82714ca7b295933a0c9c0f62b2e"
             ),
-            parse_obj_as(
-                SHA256Str,
-                "488f3b57932803bbf644593bd46d95599b1d4da1d63bc020d7ebe6f1c255f7f3",
+            TypeAdapter(SHA256Str).validate_python(
+                "488f3b57932803bbf644593bd46d95599b1d4da1d63bc020d7ebe6f1c255f7f3"
             ),
         ),
     ) -> tuple[

services/storage/tests/unit/test_dsm.py

@@ -10,7 +10,7 @@
 from faker import Faker
 from models_library.projects_nodes_io import SimcoreS3FileID
 from models_library.users import UserID
-from pydantic import ByteSize, parse_obj_as
+from pydantic import ByteSize, TypeAdapter
 from servicelib.utils import limited_gather
 from simcore_service_storage.models import FileMetaData, S3BucketName
 from simcore_service_storage.simcore_s3_dsm import SimcoreS3DataManager
@@ -31,7 +31,7 @@ async def dsm_mockup_complete_db(
     cleanup_user_projects_file_metadata: None,
     faker: Faker,
 ) -> tuple[FileMetaData, FileMetaData]:
-    file_size = parse_obj_as(ByteSize, "10Mib")
+    file_size = TypeAdapter(ByteSize).validate_python("10Mib")
     uploaded_files = await limited_gather(
         *(upload_file(file_size, faker.file_name(), None) for _ in range(2)),
         limit=2,