Implementation of feeds page (#1056)

* Implementation of feeds page

* Implemented Create, edit and delete feed

* Added feed page, fixed several issues. All working now

* Imlemented FeedAuthorization, addressed backend CSS issue and feedbacks

* missed file in previous commit

* Removed index_name from backend and UI

* missed change in prev commit

* addressed comments

* Adding restriction for min of 1 item for criteria and navigating to feeds page on deletion

* adding restrictions for arrays to have atleast 1 item - rsjf/mui specific changes

---------

Co-authored-by: Chen Wang <[email protected]>

Author: ddey2

backend/app/deps/authorization_deps.py

@@ -1,6 +1,7 @@
 from app.keycloak_auth import get_current_username, get_read_only_user
 from app.models.authorization import AuthorizationDB, RoleType
 from app.models.datasets import DatasetDB, DatasetStatus
+from app.models.feeds import FeedDB
 from app.models.files import FileDB, FileStatus
 from app.models.groups import GroupDB
 from app.models.listeners import EventListenerDB
@@ -426,6 +427,37 @@ async def __call__(
         raise HTTPException(status_code=404, detail=f"Listener {listener_id} not found")
 
 
+class FeedAuthorization:
+    """We use class dependency so that we can provide the `permission` parameter to the dependency.
+    For more info see https://fastapi.tiangolo.com/advanced/advanced-dependencies/.
+    Regular users can only see their own feeds"""
+
+    # def __init__(self, optional_arg: str = None):
+    #         self.optional_arg = optional_arg
+
+    async def __call__(
+        self,
+        feed_id: str,
+        current_user: str = Depends(get_current_username),
+        admin_mode: bool = Depends(get_admin_mode),
+        admin: bool = Depends(get_admin),
+    ):
+        # If the current user is admin and has turned on admin_mode, user has access irrespective of any role assigned
+        if admin and admin_mode:
+            return True
+
+        # Else check if current user is the creator of the feed
+        if (feed := await FeedDB.get(PydanticObjectId(feed_id))) is not None:
+            if feed.creator and feed.creator == current_user:
+                return True
+            else:
+                raise HTTPException(
+                    status_code=403,
+                    detail=f"User `{current_user} does not have permission on feed `{feed_id}`",
+                )
+        raise HTTPException(status_code=404, detail=f"Feed {feed_id} not found")
+
+
 class CheckStatus:
     """We use class dependency so that we can provide the `permission` parameter to the dependency.
     For more info see https://fastapi.tiangolo.com/advanced/advanced-dependencies/."""

backend/app/heartbeat_listener_sync.py

@@ -23,14 +23,17 @@ def callback(ch, method, properties, body):
 
     extractor_info = msg["extractor_info"]
     extractor_name = extractor_info["name"]
+    extractor_description = extractor_name
+    if "description" in extractor_info:
+        extractor_description = extractor_info["description"]
     extractor_db = EventListenerDB(
         **extractor_info, properties=ExtractorInfo(**extractor_info)
     )
 
     mongo_client = MongoClient(settings.MONGODB_URL)
     db = mongo_client[settings.MONGO_DATABASE]
 
-    # check to see if extractor alredy exists
+    # check to see if extractor already exists
     existing_extractor = db["listeners"].find_one({"name": msg["queue"]})
     if existing_extractor is not None:
         # Update existing listener
@@ -85,8 +88,8 @@ def callback(ch, method, properties, body):
                 # TODO: Who should the author be for an auto-generated feed? Currently None.
                 new_feed = FeedDB(
                     name=extractor_name,
+                    description=extractor_description,
                     search={
-                        "index_name": "file",
                         "criteria": criteria_list,
                         "mode": "or",
                     },

backend/app/models/feeds.py

@@ -13,14 +13,11 @@ class JobFeed(BaseModel):
     resources match the saved search criteria for the Feed."""
 
     name: str
+    description: str = ""
     search: SearchObject
     listeners: List[FeedListener] = []
 
 
-class FeedBase(JobFeed):
-    description: str = ""
-
-
 class FeedIn(JobFeed):
     pass
 

backend/app/models/search.py

@@ -14,13 +14,11 @@ class SearchObject(BaseModel):
     """This is a way to save a search (i.e. as a Feed).
 
     Parameters:
-        index_name -- which ES index to search
         criteria -- some number of field/operator/value tuples describing the search requirements
         mode -- and/or determines whether all of the criteria must match, or any of them
         original -- if the user originally performed a string search, their original text entry is preserved here
     """
 
-    index_name: str
     criteria: List[SearchCriteria] = []
     mode: str = "and"  # and / or
     original: Optional[str]  # original un-parsed search string

backend/app/routers/feeds.py

@@ -1,23 +1,27 @@
-from typing import List, Optional
+from typing import Optional
 
-from app.deps.authorization_deps import ListenerAuthorization
+from app.deps.authorization_deps import FeedAuthorization, ListenerAuthorization
 from app.keycloak_auth import get_current_user, get_current_username
 from app.models.feeds import FeedDB, FeedIn, FeedOut
 from app.models.files import FileOut
 from app.models.listeners import EventListenerDB, FeedListener
+from app.models.pages import Paged, _construct_page_metadata, _get_page_query
 from app.models.users import UserOut
 from app.rabbitmq.listeners import submit_file_job
 from app.routers.authentication import get_admin, get_admin_mode
 from app.search.connect import check_search_result
 from beanie import PydanticObjectId
+from beanie.operators import Or, RegEx
 from fastapi import APIRouter, Depends, HTTPException
 from pika.adapters.blocking_connection import BlockingChannel
 
 router = APIRouter()
 
 
 # TODO: Move this to MongoDB middle layer
-async def disassociate_listener_db(feed_id: str, listener_id: str):
+async def disassociate_listener_db(
+    feed_id: str, listener_id: str, allows: bool = Depends(FeedAuthorization())
+):
     """Remove a specific Event Listener from a feed. Does not delete either resource, just removes relationship.
 
     This actually performs the database operations, and can be used by any endpoints that need this functionality.
@@ -71,34 +75,90 @@ async def save_feed(
     return feed.dict()
 
 
-@router.get("", response_model=List[FeedOut])
+@router.put("/{feed_id}", response_model=FeedOut)
+async def edit_feed(
+    feed_id: str,
+    feed_in: FeedIn,
+    user=Depends(get_current_username),
+    allow: bool = Depends(FeedAuthorization()),
+):
+    """Update the information about an existing Feed..
+
+    Arguments:
+        feed_id -- UUID of the feed to be udpated
+        feed_in -- JSON object including updated information
+    """
+    feed = await FeedDB.get(PydanticObjectId(feed_id))
+    if feed:
+        # TODO: Refactor this with permissions checks etc.
+        feed_update = feed_in.dict()
+        if (
+            not feed_update["name"]
+            or not feed_update["search"]
+            or len(feed_update["listeners"]) == 0
+        ):
+            raise HTTPException(
+                status_code=400,
+                detail="Feed name/search/listeners can't be null or empty",
+            )
+            return
+        feed.description = feed_update["description"]
+        feed.name = feed_update["name"]
+        feed.search = feed_update["search"]
+        feed.listeners = feed_update["listeners"]
+        try:
+            await feed.save()
+            return feed.dict()
+        except Exception as e:
+            raise HTTPException(status_code=500, detail=e.args[0])
+    raise HTTPException(status_code=404, detail=f"listener {feed_id} not found")
+
+
+@router.get("", response_model=Paged)
 async def get_feeds(
-    name: Optional[str] = None,
+    searchTerm: Optional[str] = None,
     user=Depends(get_current_user),
     skip: int = 0,
     limit: int = 10,
+    admin=Depends(get_admin),
+    admin_mode=Depends(get_admin_mode),
 ):
     """Fetch all existing Feeds."""
-    if name is not None:
-        feeds = (
-            await FeedDB.find(FeedDB.name == name)
-            .sort(-FeedDB.created)
-            .skip(skip)
-            .limit(limit)
-            .to_list()
-        )
-    else:
-        feeds = (
-            await FeedDB.find().sort(-FeedDB.created).skip(skip).limit(limit).to_list()
+    criteria_list = []
+    if not admin or not admin_mode:
+        criteria_list.append(FeedDB.creator == user.email)
+    if searchTerm is not None:
+        criteria_list.append(
+            Or(
+                RegEx(field=FeedDB.name, pattern=searchTerm, options="i"),
+                RegEx(field=FeedDB.description, pattern=searchTerm, options="i"),
+            )
         )
 
-    return [feed.dict() for feed in feeds]
+    feeds_and_count = (
+        await FeedDB.find(
+            *criteria_list,
+        )
+        .aggregate(
+            [_get_page_query(skip, limit, sort_field="created", ascending=False)],
+        )
+        .to_list()
+    )
+    page_metadata = _construct_page_metadata(feeds_and_count, skip, limit)
+    page = Paged(
+        metadata=page_metadata,
+        data=[
+            FeedOut(id=item.pop("_id"), **item) for item in feeds_and_count[0]["data"]
+        ],
+    )
+    return page.dict()
 
 
 @router.get("/{feed_id}", response_model=FeedOut)
 async def get_feed(
     feed_id: str,
     user=Depends(get_current_user),
+    allow: bool = Depends(FeedAuthorization()),
 ):
     """Fetch an existing saved search Feed."""
     if (feed := await FeedDB.get(PydanticObjectId(feed_id))) is not None:
@@ -107,15 +167,16 @@ async def get_feed(
         raise HTTPException(status_code=404, detail=f"Feed {feed_id} not found")
 
 
-@router.delete("/{feed_id}")
+@router.delete("/{feed_id}", response_model=FeedOut)
 async def delete_feed(
     feed_id: str,
     user=Depends(get_current_user),
+    allow: bool = Depends(FeedAuthorization()),
 ):
     """Delete an existing saved search Feed."""
     if (feed := await FeedDB.get(PydanticObjectId(feed_id))) is not None:
         await feed.delete()
-        return {"deleted": feed_id}
+        return feed.dict()
     raise HTTPException(status_code=404, detail=f"Feed {feed_id} not found")
 
 
@@ -126,6 +187,7 @@ async def associate_listener(
     user=Depends(get_current_user),
     admin=Depends(get_admin),
     admin_mode=Depends(get_admin_mode),
+    allow: bool = Depends(FeedAuthorization()),
 ):
     """Associate an existing Event Listener with a Feed, e.g. so it will be triggered on new Feed results.
 

backend/app/search/connect.py

@@ -159,9 +159,9 @@ def check_search_result(es_client, file_out: FileOut, search_obj: SearchObject):
         match_list.append({"match": crit})
 
     # TODO: This will need to be more complex to support other operators
-    if search_obj.mode == "and":
+    if search_obj.mode.lower() == "and":
         subquery = {"bool": {"must": match_list}}
-    if search_obj.mode == "or":
+    if search_obj.mode.lower() == "or":
         subquery = {"bool": {"should": match_list}}
 
     # Wrap the normal criteria with restriction of file ID also