✨♻️ Storage refactoring step3 (ITISFoundation#3144)

Author: sanderegg

api/specs/storage/openapi.yaml

@@ -243,9 +243,11 @@ paths:
                 $ref: "#/components/schemas/FileMetaEnvelope"
         default:
           $ref: "#/components/responses/DefaultErrorResponse"
-    patch:
-      summary: Update file metadata
-      operationId: update_file_meta_data
+
+  /locations/{location_id}/files/{file_id}:abort:
+    post:
+      summary: Asks the server to abort the upload and revert to the last valid version if any
+      operationId: abort_upload_file
       parameters:
         - name: file_id
           in: path
@@ -263,12 +265,8 @@ paths:
           schema:
             type: string
       responses:
-        "200":
-          description: "Returns file metadata"
-          content:
-            application/json:
-              schema:
-                $ref: "#/components/schemas/FileMetaEnvelope"
+        "204":
+          description: Abort OK
         default:
           $ref: "#/components/responses/DefaultErrorResponse"
 
@@ -346,6 +344,12 @@ paths:
             application/json:
               schema:
                 $ref: "#/components/schemas/PresignedLinkEnveloped"
+          links:
+            AbortUpload:
+              operationId: abort_upload_file
+              parameters:
+                path.location_id: "$request.path.location_id"
+                path.file_id: "$request.path.file_id"
         default:
           $ref: "#/components/responses/DefaultErrorResponse"
     delete:

packages/models-library/src/models_library/api_schemas_storage.py

@@ -8,11 +8,26 @@
 
 import re
 from datetime import datetime
-from typing import List, Optional, Pattern, Union
+from enum import Enum
+from typing import Any, Optional, Pattern, Union
 from uuid import UUID
 
-from models_library.projects_nodes_io import LocationID, LocationName, StorageFileID
-from pydantic import BaseModel, ByteSize, ConstrainedStr, Extra, Field, validator
+from models_library.projects_nodes_io import (
+    LocationID,
+    LocationName,
+    NodeID,
+    SimcoreS3FileID,
+    StorageFileID,
+)
+from pydantic import (
+    BaseModel,
+    ByteSize,
+    ConstrainedStr,
+    Extra,
+    Field,
+    root_validator,
+    validator,
+)
 from pydantic.networks import AnyUrl
 
 from .basic_regex import DATCORE_DATASET_NAME_RE, S3_BUCKET_NAME_RE
@@ -127,8 +142,8 @@ class FileMetaDataGet(BaseModel):
 
     @validator("location_id", pre=True)
     @classmethod
-    def convert_from_str(cls, v):
-        if isinstance(v, str):
+    def ensure_location_is_integer(cls, v):
+        if v is not None:
             return int(v)
         return v
 
@@ -198,14 +213,46 @@ class Config:
 
 
 class FileMetaDataArray(BaseModel):
-    __root__: List[FileMetaDataGet] = []
+    __root__: list[FileMetaDataGet] = []
 
 
 # /locations/{location_id}/files/{file_id}
 
 
+class LinkType(str, Enum):
+    PRESIGNED = "PRESIGNED"
+    S3 = "S3"
+
+
 class PresignedLink(BaseModel):
     link: AnyUrl
 
 
 # /simcore-s3/
+
+
+class FoldersBody(BaseModel):
+    source: dict[str, Any] = Field(default_factory=dict)
+    destination: dict[str, Any] = Field(default_factory=dict)
+    nodes_map: dict[NodeID, NodeID] = Field(default_factory=dict)
+
+    @root_validator()
+    @classmethod
+    def ensure_consistent_entries(cls, values):
+        source_node_keys = (
+            NodeID(n) for n in values["source"].get("workbench", {}).keys()
+        )
+        if set(source_node_keys) != set(values["nodes_map"].keys()):
+            raise ValueError("source project nodes do not fit with nodes_map entries")
+        destination_node_keys = (
+            NodeID(n) for n in values["destination"].get("workbench", {}).keys()
+        )
+        if set(destination_node_keys) != set(values["nodes_map"].values()):
+            raise ValueError(
+                "destination project nodes do not fit with nodes_map values"
+            )
+        return values
+
+
+class SoftCopyBody(BaseModel):
+    link_id: SimcoreS3FileID

packages/models-library/src/models_library/basic_types.py

@@ -2,7 +2,7 @@
 
 from pydantic import conint, constr
 
-from .basic_regex import UUID_RE_BASE, VERSION_RE
+from .basic_regex import UUID_RE, VERSION_RE
 
 # port number range
 PortInt = conint(gt=0, lt=65535)
@@ -21,7 +21,7 @@
 EnvVarKey = constr(regex=r"[a-zA-Z][a-azA-Z0-9_]*")
 
 # e.g. '5c833a78-1af3-43a7-9ed7-6a63b188f4d8'
-UUIDStr = constr(regex=UUID_RE_BASE)
+UUIDStr = constr(regex=UUID_RE)
 
 
 class LogLevel(str, Enum):

packages/models-library/src/models_library/projects.py

@@ -4,7 +4,7 @@
 from copy import deepcopy
 from datetime import datetime
 from enum import Enum
-from typing import Any, Dict, List, Optional
+from typing import Any, Optional
 from uuid import UUID
 
 from pydantic import BaseModel, EmailStr, Extra, Field, HttpUrl, constr, validator
@@ -22,7 +22,7 @@
 ClassifierID = str
 
 # TODO: for some reason class Workbench(BaseModel): __root__= does not work as I thought ... investigate!
-Workbench = Dict[NodeIDStr, Node]
+Workbench = dict[NodeIDStr, Node]
 
 
 # NOTE: careful this is in sync with packages/postgres-database/src/simcore_postgres_database/models/projects.py!!!
@@ -97,6 +97,7 @@ def convert_sql_alchemy_enum(cls, v):
     class Config:
         orm_mode = True
         use_enum_values = True
+        allow_population_by_field_name = True
 
 
 class Project(BaseProjectModel):
@@ -121,15 +122,15 @@ class Project(BaseProjectModel):
         examples=["2018-07-01T11:13:43Z"],
         alias="lastChangeDate",
     )
-    access_rights: Dict[GroupIDStr, AccessRights] = Field(
+    access_rights: dict[GroupIDStr, AccessRights] = Field(
         ...,
         description="object containing the GroupID as key and read/write/execution permissions as value",
         alias="accessRights",
     )
 
     # Classification
-    tags: Optional[List[int]] = []
-    classifiers: Optional[List[ClassifierID]] = Field(
+    tags: Optional[list[int]] = []
+    classifiers: Optional[list[ClassifierID]] = Field(
         default_factory=list,
         description="Contains the reference to the project classifiers",
         examples=["some:id:to:a:classifier"],
@@ -142,20 +143,20 @@ class Project(BaseProjectModel):
     ui: Optional[StudyUI] = None
 
     # Quality
-    quality: Dict[str, Any] = Field(
+    quality: dict[str, Any] = Field(
         {}, description="stores the study quality assessment"
     )
 
     # Dev only
-    dev: Optional[Dict] = Field(description="object used for development purposes only")
+    dev: Optional[dict] = Field(description="object used for development purposes only")
 
     class Config:
         description = "Document that stores metadata, pipeline and UI setup of a study"
         title = "osparc-simcore project"
         extra = Extra.forbid
 
         @staticmethod
-        def schema_extra(schema: Dict, _model: "Project"):
+        def schema_extra(schema: dict, _model: "Project"):
             # pylint: disable=unsubscriptable-object
 
             # Patch to allow jsonschema nullable

packages/models-library/src/models_library/projects_nodes_io.py

@@ -7,9 +7,8 @@
 """
 
 import re
-from enum import IntEnum
 from pathlib import Path
-from typing import Literal, Optional, Pattern, Union
+from typing import Optional, Pattern, Union
 from uuid import UUID
 
 from pydantic import AnyUrl, BaseModel, ConstrainedStr, Extra, Field, validator
@@ -24,14 +23,8 @@ class NodeIDStr(ConstrainedStr):
     regex: Optional[Pattern[str]] = re.compile(UUID_RE)
 
 
-# NOTE: this trick is used to keep backward compatility simcore.s3 is not a valid python variable name
-Location = IntEnum(
-    value="Location",
-    names=[("simcore.s3", 0), ("SIMCORE_S3", 0), ("datcore", 1), ("DATCORE", 1)],
-)
-
-LocationID = Union[Literal[0], Literal[1]]
-LocationName = Union[Literal["simcore.s3"], Literal["datcore"]]
+LocationID = int
+LocationName = str
 
 
 class SimcoreS3FileID(ConstrainedStr):

packages/models-library/tests/test_basic_regex.py

@@ -12,7 +12,7 @@
 from models_library.basic_regex import (
     DATE_RE,
     PUBLIC_VARIABLE_NAME_RE,
-    UUID_RE_BASE,
+    UUID_RE,
     VERSION_RE,
 )
 from packaging.version import Version
@@ -30,6 +30,7 @@ def assert_match_and_get_capture(regex_str, test_str, expected) -> Optional[Sequ
         assert match is not None
         print(regex_str, "captured:", match.group(), "->", match.groups())
     else:
+        assert match
         captured = match.groups()
         assert captured == expected
         return captured
@@ -56,7 +57,7 @@ def test_VERSION_RE(version_str, expected):
     ],
 )
 def test_UUID_RE(uuid_str, expected):
-    assert_match_and_get_capture(UUID_RE_BASE, uuid_str, expected)
+    assert_match_and_get_capture(UUID_RE, uuid_str, expected)
 
 
 class webserver_timedate_utils:

packages/postgres-database/src/simcore_postgres_database/migration/versions/2cc556e5c52d_file_meta_data_remove_unused_columns_.py

@@ -0,0 +1,87 @@
+"""file_meta_data: remove unused columns, add expiration of upload
+
+Revision ID: 2cc556e5c52d
+Revises: cf3bac482ce0
+Create Date: 2022-06-26 19:12:13.478593+00:00
+
+"""
+import sqlalchemy as sa
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision = "2cc556e5c52d"
+down_revision = "cf3bac482ce0"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.add_column(
+        "file_meta_data", sa.Column("upload_expires_at", sa.DateTime(), nullable=True)
+    )
+    op.alter_column(
+        "file_meta_data", "file_id", existing_type=sa.VARCHAR(), nullable=False
+    )
+    op.drop_column("file_meta_data", "display_file_path")
+    op.drop_column("file_meta_data", "node_name")
+    op.drop_column("file_meta_data", "raw_file_path")
+    op.drop_column("file_meta_data", "project_name")
+    op.drop_column("file_meta_data", "file_name")
+    op.drop_column("file_meta_data", "file_uuid")
+    op.drop_column("file_meta_data", "user_name")
+    # ### end Alembic commands ###
+    op.create_primary_key("pk_file_meta_data", "file_meta_data", ["file_id"])
+
+
+def downgrade():
+    op.drop_constraint("pk_file_meta_data", "file_meta_data", "primary")
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.add_column(
+        "file_meta_data",
+        sa.Column("user_name", sa.VARCHAR(), autoincrement=False, nullable=True),
+    )
+    op.add_column(
+        "file_meta_data",
+        sa.Column("file_uuid", sa.VARCHAR(), autoincrement=False, nullable=True),
+    )
+    op.add_column(
+        "file_meta_data",
+        sa.Column("file_name", sa.VARCHAR(), autoincrement=False, nullable=True),
+    )
+    op.add_column(
+        "file_meta_data",
+        sa.Column("project_name", sa.VARCHAR(), autoincrement=False, nullable=True),
+    )
+    op.add_column(
+        "file_meta_data",
+        sa.Column("raw_file_path", sa.VARCHAR(), autoincrement=False, nullable=True),
+    )
+    op.add_column(
+        "file_meta_data",
+        sa.Column("node_name", sa.VARCHAR(), autoincrement=False, nullable=True),
+    )
+    op.add_column(
+        "file_meta_data",
+        sa.Column(
+            "display_file_path", sa.VARCHAR(), autoincrement=False, nullable=True
+        ),
+    )
+    op.alter_column(
+        "file_meta_data", "file_id", existing_type=sa.VARCHAR(), nullable=True
+    )
+    op.drop_column("file_meta_data", "upload_expires_at")
+    # ### end Alembic commands ###
+    conn = op.get_bind()
+    for row in conn.execute(sa.DDL("SELECT file_id FROM file_meta_data")):
+        file_id = row["file_id"]
+        conn.execute(
+            sa.DDL(
+                f"""
+UPDATE file_meta_data
+    SET file_uuid = '{file_id}'
+    WHERE file_id = '{file_id}'
+    """
+            )
+        )
+    op.create_primary_key("pk_file_meta_data", "file_meta_data", ["file_uuid"])

packages/postgres-database/src/simcore_postgres_database/models/file_meta_data.py

@@ -5,21 +5,14 @@
 file_meta_data = sa.Table(
     "file_meta_data",
     metadata,
-    sa.Column("file_uuid", sa.String(), primary_key=True),
     sa.Column("location_id", sa.String()),
     sa.Column("location", sa.String()),
     sa.Column("bucket_name", sa.String()),
     sa.Column("object_name", sa.String()),
     sa.Column("project_id", sa.String()),
-    sa.Column("project_name", sa.String()),
     sa.Column("node_id", sa.String()),
-    sa.Column("node_name", sa.String()),
-    sa.Column("file_name", sa.String()),
     sa.Column("user_id", sa.String()),
-    sa.Column("user_name", sa.String()),
-    sa.Column("file_id", sa.String()),
-    sa.Column("raw_file_path", sa.String()),
-    sa.Column("display_file_path", sa.String()),
+    sa.Column("file_id", sa.String(), primary_key=True),
     sa.Column("created_at", sa.String()),
     sa.Column("last_modified", sa.String()),
     sa.Column("file_size", sa.BigInteger()),
@@ -38,4 +31,7 @@
         doc="If true, this file is a soft link."
         "i.e. is another entry with the same object_name",
     ),
+    sa.Column(
+        "upload_expires_at", sa.DateTime(), nullable=True, doc="Timestamp of expiration"
+    ),
 )