webserver api

Author: sanderegg

packages/models-library/src/models_library/api_schemas_storage/storage_schemas.py

@@ -8,13 +8,10 @@
 
 from datetime import datetime
 from enum import Enum
-
-# /data-export
+from pathlib import Path
 from typing import Annotated, Any, Literal, Self, TypeAlias
 from uuid import UUID
 
-from models_library.projects import ProjectID
-from models_library.users import UserID
 from pydantic import (
     BaseModel,
     ByteSize,
@@ -28,16 +25,23 @@
 )
 from pydantic.networks import AnyUrl
 
-from ..basic_regex import DATCORE_DATASET_NAME_RE, S3_BUCKET_NAME_RE
+from ..basic_regex import (
+    DATCORE_COLLECTION_NAME_RE,
+    DATCORE_DATASET_NAME_RE,
+    DATCORE_FILE_ID_RE,
+    S3_BUCKET_NAME_RE,
+)
 from ..basic_types import SHA256Str
 from ..generics import ListModel
+from ..projects import ProjectID
 from ..projects_nodes_io import (
     LocationID,
     LocationName,
     NodeID,
     SimcoreS3FileID,
     StorageFileID,
 )
+from ..users import UserID
 
 ETag: TypeAlias = str
 
@@ -46,6 +50,12 @@
 DatCoreDatasetName: TypeAlias = Annotated[
     str, StringConstraints(pattern=DATCORE_DATASET_NAME_RE)
 ]
+DatCoreCollectionName: TypeAlias = Annotated[
+    str, StringConstraints(pattern=DATCORE_COLLECTION_NAME_RE)
+]
+DatCorePackageName: TypeAlias = Annotated[
+    str, StringConstraints(pattern=DATCORE_FILE_ID_RE)
+]
 
 
 # /
@@ -263,9 +273,6 @@ class FileMetaDataArray(RootModel[list[FileMetaDataGet]]):
     root: list[FileMetaDataGet] = Field(default_factory=list)
 
 
-# /locations/{location_id}/files/{file_id}
-
-
 class LinkType(str, Enum):
     PRESIGNED = "PRESIGNED"
     S3 = "S3"
@@ -370,3 +377,53 @@ def ensure_consistent_entries(self: Self) -> Self:
 
 class SoftCopyBody(BaseModel):
     link_id: SimcoreS3FileID
+
+
+class PathMetaDataGet(BaseModel):
+    path: Annotated[Path, Field(description="the path to the current path")]
+    display_path: Annotated[
+        Path, Field(description="the path to display with UUID replaced")
+    ]
+
+    file_meta_data: Annotated[
+        FileMetaDataGet | None,
+        Field(description="if filled, this is the file meta data of the s3 object"),
+    ] = None
+
+    model_config = ConfigDict(
+        extra="forbid",
+        json_schema_extra={
+            "examples": [
+                # ls no filter
+                {
+                    "path": "f8da77a9-24b9-4eab-aee7-1f0608da1e3e",
+                    "display_path": "my amazing project",
+                },
+                # ls f8da77a9-24b9-4eab-aee7-1f0608da1e3e
+                {
+                    "path": "f8da77a9-24b9-4eab-aee7-1f0608da1e3e/2f94f80f-633e-4dfa-a983-226b7babe3d7",
+                    "display_path": "my amazing project/awesome node",
+                },
+                # ls f8da77a9-24b9-4eab-aee7-1f0608da1e3e/2f94f80f-633e-4dfa-a983-226b7babe3d7
+                {
+                    "path": "f8da77a9-24b9-4eab-aee7-1f0608da1e3e/2f94f80f-633e-4dfa-a983-226b7babe3d7/outputs",
+                    "display_path": "my amazing project/awesome node/outputs",
+                },
+                # ls f8da77a9-24b9-4eab-aee7-1f0608da1e3e/2f94f80f-633e-4dfa-a983-226b7babe3d7/outputs
+                {
+                    "path": "f8da77a9-24b9-4eab-aee7-1f0608da1e3e/2f94f80f-633e-4dfa-a983-226b7babe3d7/outputs/output5",
+                    "display_path": "my amazing project/awesome node/outputs/output5",
+                },
+                # ls f8da77a9-24b9-4eab-aee7-1f0608da1e3e/2f94f80f-633e-4dfa-a983-226b7babe3d7/outputs/output_5
+                {
+                    "path": f"f8da77a9-24b9-4eab-aee7-1f0608da1e3e/2f94f80f-633e-4dfa-a983-226b7babe3d7/outputs/output5/{FileMetaDataGet.model_config['json_schema_extra']['examples'][0]['file_name']}",  # type: ignore[index, call-overload]
+                    "display_path": f"my amazing project/awesome node/outputs/output5/{FileMetaDataGet.model_config['json_schema_extra']['examples'][0]['file_name']}",  # type: ignore[index, call-overload]
+                    "file_meta_data": FileMetaDataGet.model_config["json_schema_extra"][  # type: ignore[index]
+                        "examples"
+                    ][
+                        0  # type: ignore[index]
+                    ],
+                },
+            ]
+        },
+    )

packages/models-library/src/models_library/api_schemas_webserver/storage.py

@@ -2,6 +2,8 @@
 from pathlib import Path
 from typing import Any
 
+from pydantic import BaseModel
+
 from ..api_schemas_rpc_async_jobs.async_jobs import (
     AsyncJobGet,
     AsyncJobId,
@@ -11,10 +13,19 @@
 from ..api_schemas_storage.data_export_async_jobs import DataExportTaskStartInput
 from ..progress_bar import ProgressReport
 from ..projects_nodes_io import LocationID
+from ..rest_pagination import CursorQueryParameters
 from ..users import UserID
 from ._base import InputSchema, OutputSchema
 
 
+class StorageLocationPathParams(BaseModel):
+    location_id: LocationID
+
+
+class ListPathsQueryParams(InputSchema, CursorQueryParameters):
+    file_filter: Path | None = None
+
+
 class DataExportPost(InputSchema):
     paths: list[Path]
 

packages/models-library/src/models_library/basic_regex.py

@@ -1,8 +1,8 @@
-""" Regular expressions patterns to build pydantic contrained strings
+"""Regular expressions patterns to build pydantic contrained strings
 
-    - Variants of the patterns with 'Named Groups' captured are suffixed with NG_RE
+- Variants of the patterns with 'Named Groups' captured are suffixed with NG_RE
 
-    SEE tests_basic_regex.py for examples
+SEE tests_basic_regex.py for examples
 """
 # TODO: for every pattern we should have a formatter function
 # NOTE: some sites to manualy check ideas
@@ -56,6 +56,7 @@
 # Datcore file ID
 DATCORE_FILE_ID_RE = rf"^N:package:{UUID_RE_BASE}$"
 DATCORE_DATASET_NAME_RE = rf"^N:dataset:{UUID_RE_BASE}$"
+DATCORE_COLLECTION_NAME_RE = rf"^N:collection:{UUID_RE_BASE}$"
 
 
 TWILIO_ALPHANUMERIC_SENDER_ID_RE = r"(?!^\d+$)^[a-zA-Z0-9\s]{2,11}$"

packages/models-library/src/models_library/rest_pagination.py

@@ -31,6 +31,23 @@
 ).validate_python(20)
 
 
+class CursorQueryParameters(RequestParameters):
+    """Use as pagination options in query parameters"""
+
+    size: PageLimitInt = Field(
+        default=TypeAdapter(PageLimitInt).validate_python(
+            DEFAULT_NUMBER_OF_ITEMS_PER_PAGE
+        ),
+        description="maximum number of items to return (pagination)",
+    )
+    cursor: Annotated[
+        str | None,
+        Field(
+            description="unique identifier that represent the position in the dataset"
+        ),
+    ] = None
+
+
 class PageQueryParameters(RequestParameters):
     """Use as pagination options in query parameters"""