🩹 Adds query option to avoid timeout for storage metadata sync (ITISFoundation#2428)

🩹 fix sync between storage's metadata table and s3 objects:
  - PROBLEM: current sync makes webserver handler time out when s3 holds a lot of data.
  - SOLUTION: improves check speed in S3 and adds option to fire&forget in query
✨ adds new user role ADMIN. Resync operation can only be done by ADMIN users

Author: pcrespov

api/specs/storage/openapi.yaml

@@ -113,6 +113,12 @@ paths:
           schema:
             type: boolean
             default: true
+        - name: fire_and_forget
+          in: query
+          required: false
+          schema:
+            type: boolean
+            default: false
       responses:
         "200":
           description: An object containing added, changed and removed paths
@@ -671,6 +677,10 @@ components:
       required:
         - removed
       properties:
+        dry_run:
+          type: boolean
+        fire_and_forget:
+          type: boolean
         removed:
           type: array
           items:

api/specs/webserver/components/schemas/locations.yaml

@@ -42,6 +42,10 @@ TableSynchronisation:
   required:
     - removed
   properties:
+    dry_run:
+      type: boolean
+    fire_and_forget:
+      type: boolean
     removed:
       type: array
       items:

api/specs/webserver/openapi-storage.yaml

@@ -33,6 +33,12 @@ paths:
           schema:
             type: boolean
             default: true
+        - name: fire_and_forget
+          in: query
+          required: false
+          schema:
+            type: boolean
+            default: false
       responses:
         "200":
           description: An object containing added, changed and removed paths

packages/postgres-database/src/simcore_postgres_database/migration/versions/c2d3acc313e1_adds_admin_to_userrole_enum.py

@@ -0,0 +1,34 @@
+"""Adds ADMIN to userrole enum
+
+Revision ID: c2d3acc313e1
+Revises: 87a7b69f6723
+Create Date: 2021-07-12 13:05:05.782876+00:00
+
+"""
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision = "c2d3acc313e1"
+down_revision = "87a7b69f6723"
+branch_labels = None
+depends_on = None
+
+
+# SEE https://medium.com/makimo-tech-blog/upgrading-postgresqls-enum-type-with-sqlalchemy-using-alembic-migration-881af1e30abe
+
+
+def upgrade():
+    with op.get_context().autocommit_block():
+        op.execute("ALTER TYPE userrole ADD VALUE 'ADMIN'")
+
+
+def downgrade():
+    op.execute("ALTER TYPE userrole RENAME TO userrole_old")
+    op.execute("CREATE TYPE userrole AS ENUM('ANONYMOUS', 'GUEST', 'USER', 'TESTER')")
+    op.execute(
+        (
+            "ALTER TABLE users ALTER COLUMN role TYPE userrole USING "
+            "role::text::userrole"
+        )
+    )
+    op.execute("DROP TYPE userrole_old")

packages/postgres-database/src/simcore_postgres_database/models/users.py

@@ -14,7 +14,7 @@
 
 
 class UserRole(Enum):
-    """ SORTED enumeration of user roles
+    """SORTED enumeration of user roles
 
     A role defines a set of privileges the user can perform
     Roles are sorted from lower to highest privileges
@@ -25,6 +25,7 @@ class UserRole(Enum):
     USER      : Registered user. Basic permissions to use the platform [default]
     TESTER    : Upgraded user. First level of super-user with privileges to test the framework.
                 Can use everything but does not have an effect in other users or actual data
+    ADMIN     : Framework admin.
 
     See security_access.py
     """
@@ -33,6 +34,7 @@ class UserRole(Enum):
     GUEST = "GUEST"
     USER = "USER"
     TESTER = "TESTER"
+    ADMIN = "ADMIN"
 
     @classmethod
     def super_users(cls):
@@ -43,9 +45,9 @@ def super_users(cls):
 
 class UserStatus(Enum):
     """
-        pending: user registered but not confirmed
-        active: user is confirmed and can use the platform
-        banned: user is not authorized
+    pending: user registered but not confirmed
+    active: user is confirmed and can use the platform
+    banned: user is not authorized
     """
 
     CONFIRMATION_PENDING = "PENDING"
@@ -99,7 +101,7 @@ class UserStatus(Enum):
 CREATE TRIGGER user_modification
 AFTER INSERT OR UPDATE OR DELETE ON users
     FOR EACH ROW
-    EXECUTE PROCEDURE set_user_groups();    
+    EXECUTE PROCEDURE set_user_groups();
 """
 )
 
@@ -130,5 +132,7 @@ class UserStatus(Enum):
 
 sa.event.listen(users, "after_create", set_user_groups_procedure)
 sa.event.listen(
-    users, "after_create", new_user_trigger,
+    users,
+    "after_create",
+    new_user_trigger,
 )

services/storage/requirements/_base.in

@@ -1,26 +1,27 @@
-# WARNING: manually disabled urllib3 contraint to upgrade minio
+#
+# Specifies dependencies required by 'storage' service
+#
+
 --constraint ../../../requirements/constraints.txt
 
 --requirement ../../../packages/models-library/requirements/_base.in
 --requirement ../../../packages/postgres-database/requirements/_base.in
 --requirement ../../../packages/service-library/requirements/_base.in
 --requirement ../../../packages/settings-library/requirements/_base.in
 
-
-aiobotocore
-
+# server
 aiohttp
 aiohttp-swagger[performance]
 
+# s3 storage
+aiobotocore
+minio
+
+# i/o + db
 aiofiles
 aiopg[sa]
 
-tenacity
+# misc
 semantic_version
+tenacity
 typer
-minio
-urllib3
-
-#
-# FIXME: pip copiling these requirements wil fail due to incompatible contraints on urllib3 (ongoing in weekely mainteinance)
-#

services/storage/requirements/_test.in

@@ -1,10 +1,8 @@
 #
-# Specifies dependencies required to run 'storage'
+# Specifies dependencies required to TEST 'storage'
 #
+
 --constraint ../../../requirements/constraints.txt
-# Adds base AS CONSTRAINT specs, not requirement.
-#  - Resulting _text.txt is a frozen list of EXTRA packages for testing, besides _base.txt
-#
 --constraint _base.txt
 
 # testing
@@ -16,17 +14,17 @@ pytest-instafail
 pytest-mock
 pytest-runner
 pytest-sugar
-pylint
-python-dotenv
 
 # test coverage
 coverage
 coveralls
 codecov
+
+# fixtures
 faker
+pandas
+pylint
+python-dotenv
 
-# remote debugging
+# remote debugger
 ptvsd
-
-# utils
-pandas

services/storage/src/simcore_service_storage/api/v0/openapi.yaml

@@ -111,6 +111,12 @@ paths:
           schema:
             type: boolean
             default: true
+        - name: fire_and_forget
+          in: query
+          required: false
+          schema:
+            type: boolean
+            default: false
       responses:
         '200':
           description: 'An object containing added, changed and removed paths'
@@ -638,6 +644,10 @@ components:
       required:
         - removed
       properties:
+        dry_run:
+          type: boolean
+        fire_and_forget:
+          type: boolean
         removed:
           type: array
           items:

services/storage/src/simcore_service_storage/dsm.py

@@ -16,6 +16,7 @@
 import aiobotocore
 import attr
 import sqlalchemy as sa
+from aiobotocore.client import AioBaseClient
 from aiobotocore.session import AioSession, ClientCreatorContext
 from aiohttp import web
 from aiopg.sa import Engine
@@ -154,6 +155,8 @@ class DataStorageManager:
     def _create_client_context(self) -> ClientCreatorContext:
         assert hasattr(self.session, "create_client")
         # pylint: disable=no-member
+
+        # SEE API in https://botocore.amazonaws.com/v1/documentation/api/latest/reference/services/s3.html
         return self.session.create_client(
             "s3",
             endpoint_url=self.s3_client.endpoint_url,
@@ -898,7 +901,7 @@ async def delete_file(self, user_id: str, location: str, file_uuid: str):
 
     async def delete_project_simcore_s3(
         self, user_id: str, project_id: str, node_id: Optional[str] = None
-    ) -> web.Response:
+    ) -> Optional[web.Response]:
 
         """Deletes all files from a given node in a project in simcore.s3 and updated db accordingly.
         If node_id is not given, then all the project files db entries are deleted.
@@ -1013,38 +1016,55 @@ async def create_soft_link(
     async def synchronise_meta_data_table(
         self, location: str, dry_run: bool
     ) -> Dict[str, Any]:
-        sync_results = {"removed": []}
+
+        to_remove = []
+
+        assert (  # nosec
+            location == SIMCORE_S3_STR
+        ), "Only with s3, no other sync implemented"  # nosec
+
         if location == SIMCORE_S3_STR:
-            # NOTE: only valid for Simcore, since datcore data is not in the database table
+
+            # NOTE: only valid for simcore, since datcore data is not in the database table
             # let's get all the files in the table
             logger.warning(
                 "synchronisation of database/s3 storage started, this will take some time..."
             )
+
             async with self.engine.acquire() as conn, self._create_client_context() as s3_client:
-                number_of_rows_in_db = await conn.scalar(file_meta_data.count())
+
+                number_of_rows_in_db = await conn.scalar(file_meta_data.count()) or 0
                 logger.warning(
-                    "total number of entries to check %d",
+                    "Total number of entries to check %d",
                     number_of_rows_in_db,
                 )
 
-                async for row in conn.execute(file_meta_data.select()):
+                assert isinstance(s3_client, AioBaseClient)  # nosec
+
+                async for row in conn.execute(
+                    sa.select([file_meta_data.c.object_name])
+                ):
                     s3_key = row.object_name  # type: ignore
 
                     # now check if the file exists in S3
-                    try:
-                        await s3_client.get_object(
-                            Bucket=self.simcore_bucket_name,
-                            Key=s3_key,
-                        )
-                    except s3_client.exceptions.NoSuchKey:
-                        # this file does not exist
-                        sync_results["removed"].append(s3_key)
+                    # SEE https://www.peterbe.com/plog/fastest-way-to-find-out-if-a-file-exists-in-s3
+                    response = await s3_client.list_objects_v2(
+                        Bucket=self.simcore_bucket_name, Prefix=s3_key
+                    )
+                    if response.get("KeyCount", 0) == 0:  # this file does not exist
+                        to_remove.append(s3_key)
 
+                logger.info(
+                    "%s %d entries ",
+                    "Would delete" if dry_run else "Deleting",
+                    len(to_remove),
+                )
                 if not dry_run:
                     await conn.execute(
                         file_meta_data.delete().where(
-                            file_meta_data.c.object_name.in_(sync_results["removed"])
+                            file_meta_data.c.object_name.in_(to_remove)
                         )
                     )
+                    logger.info("Deleted %s orphan items", len(to_remove))
 
-        return sync_results
+        return {"removed": to_remove}