872 refactoring pagination (#894)

* might work

* working now

* model working now

* everything in dataset should work

* facet skip limit works

* wire pagination to explore page

* set model; but can't make id work

* user has quirks but kinda works

* files pagination

* revert dataset.tsx change

* file not working

* need to replace pyObjectId with pydanticObjectId otherwise the objectId can't be json serialized

* update to pydanticObjectId

* clean up files component

* file and datasets should all work now

* toggle user works

* fix everything with search user

* fix group pagination

* fix share group

* file pagination should only stay in file page

* apikey pagination works

* metadata defintion refactored

* listener jobs not quite working

* listener pagination finished

* rewrite sort

* replace all pyObjectId to pydanticObjectId

* extraction history hooked with backend

* fix pytest

* sorting works

* black

* black formatter

* fix bug

* more metadata bugs

* update pages

* moving to config

* metadata plural to singualar

* summarize page metadata into a function

* use metadata construction function in dataset

* upgrade black to latest version see if it fixes the linting issue

* Revert "upgrade black to latest version see if it fixes the linting issue"

This reverts commit aa660df.

* pin black action version to 23.3.0 to match pipfile

* black

* 896 folder plus file pagination (#903)

* fix folder pagination

* fix metadata pagination bug

* fix another one

* flipping logic clean up

* frontend pagination is impossible...

* Revert "frontend pagination is impossible..."

This reverts commit 41702ac.

* the view is working

* redux for folder and files

* listing works

* fix query bug

* nested folder correct now

* allow fancy sorting

* fix subfolder and file pagination issue

* remove outdated redux action and states

* black

* adjust file folder per page

* 901 pagination update with public datasets (#907)

* fix folder pagination

* fix metadata pagination bug

* fix another one

* flipping logic clean up

* frontend pagination is impossible...

* Revert "frontend pagination is impossible..."

This reverts commit 41702ac.

* the view is working

* redux for folder and files

* listing works

* fix query bug

* nested folder correct now

* allow fancy sorting

* fix subfolder and file pagination issue

* remove outdated redux action and states

* black

* public endpoint pagination conversion

* public folder paths not working

* fix public folder path

* black

* use utility function to construct page metadata

* small fix, call to a method that did not exist for public datasets

---------

Co-authored-by: toddn <[email protected]>

Author: longshuicy

.github/workflows/black.yml

@@ -21,4 +21,4 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v2
-      - uses: psf/black@stable
+      - uses: psf/black@23.3.0

backend/app/deps/authorization_deps.py

@@ -8,7 +8,6 @@
 from app.models.files import FileOut, FileDB, FileStatus
 from app.models.groups import GroupOut, GroupDB
 from app.models.metadata import MetadataDB
-from app.models.pyobjectid import PyObjectId
 from app.routers.authentication import get_admin
 from app.routers.authentication import get_admin_mode
 
@@ -52,7 +51,7 @@ async def get_role(
         return RoleType.OWNER
 
     authorization = await AuthorizationDB.find_one(
-        AuthorizationDB.dataset_id == PyObjectId(dataset_id),
+        AuthorizationDB.dataset_id == PydanticObjectId(dataset_id),
         Or(
             AuthorizationDB.creator == current_user,
             AuthorizationDB.user_ids == current_user,
@@ -207,7 +206,7 @@ async def __call__(
 
         # Else check role assigned to the user
         authorization = await AuthorizationDB.find_one(
-            AuthorizationDB.dataset_id == PyObjectId(dataset_id),
+            AuthorizationDB.dataset_id == PydanticObjectId(dataset_id),
             Or(
                 AuthorizationDB.creator == current_user,
                 AuthorizationDB.user_ids == current_user,

backend/app/main.py

@@ -15,6 +15,7 @@
 from app.models.errors import ErrorDB
 from app.models.feeds import FeedDB
 from app.models.files import FileDB, FileVersionDB, FileDBViewList
+from app.models.folder_and_file import FolderFileViewList
 from app.models.folders import FolderDB, FolderDBViewList
 from app.models.groups import GroupDB
 from app.models.listeners import (
@@ -269,6 +270,7 @@ async def startup_beanie():
             VisualizationConfigDB,
             VisualizationDataDB,
             ThumbnailDB,
+            FolderFileViewList,
         ],
         recreate_views=True,
     )

backend/app/models/authorization.py

@@ -1,10 +1,9 @@
 from datetime import datetime
 from enum import Enum
 
-from beanie import Document
+from beanie import Document, PydanticObjectId
 from charset_normalizer.md import List
 from pydantic import BaseModel, EmailStr, Field
-from app.models.pyobjectid import PyObjectId
 
 
 class RoleType(str, Enum):
@@ -34,10 +33,10 @@ class AuthorizationBase(BaseModel):
     group_ids are kept for convenience (adding/removing users in batch) but user_ids list MUST be kept current.
     """
 
-    dataset_id: PyObjectId
+    dataset_id: PydanticObjectId
     user_ids: List[EmailStr] = []
     role: RoleType
-    group_ids: List[PyObjectId] = []
+    group_ids: List[PydanticObjectId] = []
 
     class Config:
         # required for Enum to properly work
@@ -59,18 +58,18 @@ class Config:
 class AuthorizationFile(BaseModel):
     # TODO: This should be PyObjectId = Field(default_factory=PyObjectId). Need to figure out why can't create instance
     #  in `routers.authorization.get_dataset_role()`.
-    file_id: PyObjectId
+    file_id: PydanticObjectId
     user_ids: List[EmailStr] = []
     role: RoleType
-    group_ids: List[PyObjectId] = []
+    group_ids: List[PydanticObjectId] = []
 
     class Config:
         # required for Enum to properly work
         use_enum_values = True
 
 
 class AuthorizationMetadata(BaseModel):
-    metadata_id: PyObjectId
+    metadata_id: PydanticObjectId
     user_id: EmailStr
     role: RoleType
 

backend/app/models/files.py

@@ -1,12 +1,11 @@
 from datetime import datetime
-from enum import Enum
-from typing import Optional, List
 from enum import Enum, auto
+from typing import Optional, List
+
 from beanie import Document, View, PydanticObjectId
 from pydantic import Field, BaseModel
 
 from app.models.authorization import AuthorizationDB
-from app.models.pyobjectid import PyObjectId
 from app.models.users import UserOut
 
 
@@ -78,15 +77,16 @@ class FileDB(Document, FileBase):
     created: datetime = Field(default_factory=datetime.utcnow)
     version_id: str = "N/A"
     version_num: int = 0
-    dataset_id: PyObjectId
-    folder_id: Optional[PyObjectId]
+    dataset_id: PydanticObjectId
+    folder_id: Optional[PydanticObjectId]
     views: int = 0
     downloads: int = 0
     bytes: int = 0
     content_type: ContentType = ContentType()
     thumbnail_id: Optional[PydanticObjectId] = None
     storage_type: StorageType = StorageType.MINIO
     storage_path: Optional[str]  # store URL or file path depending on storage_type
+    object_type: str = "file"
 
     class Settings:
         name = "files"
@@ -100,8 +100,8 @@ class FileDBViewList(View, FileBase):
     id: PydanticObjectId = Field(None, alias="_id")  # necessary for Views
     version_id: str = "N/A"
     version_num: int = 0
-    dataset_id: PyObjectId
-    folder_id: Optional[PyObjectId]
+    dataset_id: PydanticObjectId
+    folder_id: Optional[PydanticObjectId]
     creator: UserOut
     created: datetime = Field(default_factory=datetime.utcnow)
     modified: datetime = Field(default_factory=datetime.utcnow)

backend/app/models/folder_and_file.py

@@ -0,0 +1,54 @@
+from datetime import datetime
+from typing import Optional, List
+
+from beanie import View, PydanticObjectId
+from pydantic import Field
+
+from app.models.authorization import AuthorizationDB
+from app.models.files import FileDB, FileBase, ContentType
+from app.models.users import UserOut
+
+
+class FolderFileViewList(View, FileBase):
+    # common field
+    object_type: str = Field(None, alias="object_type")  # necessary for Views
+    id: PydanticObjectId = Field(None, alias="_id")  # necessary for Views
+    dataset_id: PydanticObjectId
+    creator: UserOut
+    created: datetime = Field(default_factory=datetime.utcnow)
+    modified: datetime = Field(default_factory=datetime.utcnow)
+    name: str = "N/A"
+
+    # file
+    version_id: str = "N/A"
+    version_num: int = 0
+    folder_id: Optional[PydanticObjectId]
+    auth: List[AuthorizationDB]
+    bytes: int = 0
+    content_type: ContentType = ContentType()
+    thumbnail_id: Optional[PydanticObjectId] = None
+
+    # folder
+    parent_folder: Optional[PydanticObjectId]
+    auth: List[AuthorizationDB]
+
+    class Settings:
+        source = FileDB
+        name = "folders_files_view"
+        pipeline = [
+            {
+                "$lookup": {
+                    "from": "authorization",
+                    "localField": "dataset_id",
+                    "foreignField": "dataset_id",
+                    "as": "auth",
+                }
+            },
+            {"$addFields": {"object_type": "file"}},
+            {
+                "$unionWith": {
+                    "coll": "folders",
+                    "pipeline": [{"$addFields": {"object_type": "folder"}}],
+                }
+            },
+        ]

backend/app/models/folders.py

@@ -5,7 +5,6 @@
 from pydantic import Field, BaseModel
 
 from app.models.authorization import AuthorizationDB
-from app.models.pyobjectid import PyObjectId
 from app.models.users import UserOut
 
 
@@ -14,24 +13,25 @@ class FolderBase(BaseModel):
 
 
 class FolderIn(FolderBase):
-    parent_folder: Optional[PyObjectId]
+    parent_folder: Optional[PydanticObjectId]
 
 
 class FolderDB(Document, FolderBase):
-    dataset_id: PyObjectId
-    parent_folder: Optional[PyObjectId]
+    dataset_id: PydanticObjectId
+    parent_folder: Optional[PydanticObjectId]
     creator: UserOut
     created: datetime = Field(default_factory=datetime.utcnow)
     modified: datetime = Field(default_factory=datetime.utcnow)
+    object_type: str = "folder"
 
     class Settings:
         name = "folders"
 
 
 class FolderDBViewList(View, FolderBase):
     id: PydanticObjectId = Field(None, alias="_id")  # necessary for Views
-    dataset_id: PyObjectId
-    parent_folder: Optional[PyObjectId]
+    dataset_id: PydanticObjectId
+    parent_folder: Optional[PydanticObjectId]
     creator: UserOut
     created: datetime = Field(default_factory=datetime.utcnow)
     modified: datetime = Field(default_factory=datetime.utcnow)

backend/app/models/listeners.py

@@ -9,7 +9,6 @@
 from app.config import settings
 from app.models.authorization import AuthorizationDB
 from app.models.mongomodel import MongoDBRef
-from app.models.pyobjectid import PyObjectId
 from app.models.users import UserOut
 
 
@@ -96,7 +95,7 @@ class FeedListener(BaseModel):
     """This is a shorthand POST class for associating an existing EventListener with a Feed. The automatic flag determines
     whether the Feed will automatically send new matches to the Event Listener."""
 
-    listener_id: PyObjectId
+    listener_id: PydanticObjectId
     automatic: bool  # Listeners can trigger automatically or not on a per-feed basis.
 
 

backend/app/models/mongomodel.py

@@ -1,9 +1,9 @@
 from typing import Optional
-from datetime import datetime
+
+from beanie import PydanticObjectId
 from bson import ObjectId
 from bson.errors import InvalidId
-from pydantic import BaseModel, BaseConfig, Field
-from app.models.pyobjectid import PyObjectId
+from pydantic import BaseModel
 
 
 class OID(str):
@@ -25,5 +25,5 @@ def __modify_schema__(cls, field_schema):
 
 class MongoDBRef(BaseModel):
     collection: str
-    resource_id: PyObjectId
+    resource_id: PydanticObjectId
     version: Optional[int]

backend/app/models/pages.py

@@ -0,0 +1,60 @@
+from beanie.odm.enums import SortDirection
+from pydantic import BaseModel
+
+
+class PageMetadata(BaseModel):
+    total_count: int = 0
+    skip: int = 0
+    limit: int = 0
+
+
+class Paged(BaseModel):
+    metadata: PageMetadata
+    # order matters pydantic will match the first type that matches
+    # data: Union[List[FileOut], List[DatasetOut], List[GroupOut], List[UserOut], List[FeedOut],
+    # List[EventListenerJobOut], List[MetadataOut], List[MetadataDefinitionOut], List[EventListenerOut]]
+    data: list = []
+
+
+def _get_page_query(
+    skip, limit, sort_field="created", ascending=True, sort_clause=None
+):
+    if sort_clause is not None:
+        return {
+            "$facet": {
+                "metadata": [{"$count": "total_count"}],
+                "data": [
+                    sort_clause,
+                    {"$skip": skip},
+                    {"$limit": limit},
+                ],
+            },
+        }
+    else:
+        return {
+            "$facet": {
+                "metadata": [{"$count": "total_count"}],
+                "data": [
+                    {
+                        "$sort": {
+                            sort_field: SortDirection.ASCENDING
+                            if ascending
+                            else SortDirection.DESCENDING
+                        }
+                    },
+                    {"$skip": skip},
+                    {"$limit": limit},
+                ],
+            },
+        }
+
+
+def _construct_page_metadata(items_and_counts, skip, limit):
+    if len(items_and_counts[0]["metadata"]) > 0:
+        page_metadata = PageMetadata(
+            **items_and_counts[0]["metadata"][0], skip=skip, limit=limit
+        )
+    else:
+        page_metadata = PageMetadata(skip=skip, limit=limit)
+
+    return page_metadata