users unable to search files (#1076)

* files were not indexed in filesMultiple endpoint in datasets

* index files when a new user is added

* index files after removing user role

* new method - indexes all files in a dataset
this should save some time as often every file needs to be indexed after the dataset

* indexing files when we change group permissions

* fix formatting

* found bugs
when adding a user to a group, elasticsearch is not updated
when adding a group, if there were no read only users and empty list was inserted into the index

* indexing files

* temorarily removing pre commit hooks
fixing indexing for files

* using local shellcheck does not work

* removing unused import

* pre-commit

* ran pre-commit
resolved conflicts

* ran pre commit

Author: tcnichol

backend/app/routers/authorization.py

@@ -21,10 +21,11 @@
     GroupAndRole,
     UserAndRole,
 )
+from app.models.files import FileDB, FileOut
 from app.models.groups import GroupDB
 from app.models.users import UserDB
 from app.routers.authentication import get_admin, get_admin_mode
-from app.search.index import index_dataset
+from app.search.index import index_dataset, index_dataset_files
 from beanie import PydanticObjectId
 from beanie.operators import In, Or
 from bson import ObjectId
@@ -39,6 +40,7 @@ async def save_authorization(
     dataset_id: str,
     authorization_in: AuthorizationBase,
     user=Depends(get_current_username),
+    es=Depends(get_elasticsearchclient),
     allow: bool = Depends(Authorization("editor")),
 ):
     """Save authorization info in Mongo. This is a triple of dataset_id/user_id/role/group_id."""
@@ -61,6 +63,7 @@ async def save_authorization(
         **authorization_in.dict(), creator=user, user_ids=user_ids
     )
     await authorization.insert()
+    await index_dataset_files(es, dataset_id, update=True)
     return authorization.dict()
 
 
@@ -205,6 +208,7 @@ async def set_dataset_group_role(
                     await index_dataset(
                         es, DatasetOut(**dataset.dict()), auth_db.user_ids
                     )
+                    await index_dataset_files(es, str(dataset_id))
                     if len(readonly_user_ids) > 0:
                         readonly_auth_db = AuthorizationDB(
                             creator=user_id,
@@ -217,6 +221,7 @@ async def set_dataset_group_role(
                         await index_dataset(
                             es, DatasetOut(**dataset.dict()), readonly_auth_db.user_ids
                         )
+                        await index_dataset_files(es, str(dataset_id), update=True)
                 return auth_db.dict()
             else:
                 # Create new role entry for this dataset
@@ -228,27 +233,33 @@ async def set_dataset_group_role(
                     else:
                         user_ids.append(u.user.email)
                 # add the users who get the role
-                auth_db = AuthorizationDB(
-                    creator=user_id,
-                    dataset_id=PydanticObjectId(dataset_id),
-                    role=role,
-                    group_ids=[PydanticObjectId(group_id)],
-                    user_ids=user_ids,
-                )
-                readonly_auth_db = AuthorizationDB(
-                    creator=user_id,
-                    dataset_id=PydanticObjectId(dataset_id),
-                    role=RoleType.VIEWER,
-                    group_ids=[PydanticObjectId(group_id)],
-                    user_ids=readonly_user_ids,
-                )
-                # if there are read only users add them with the role of viewer
-                await auth_db.insert()
-                await index_dataset(es, DatasetOut(**dataset.dict()), auth_db.user_ids)
-                await readonly_auth_db.insert()
-                await index_dataset(
-                    es, DatasetOut(**dataset.dict()), readonly_auth_db.user_ids
-                )
+                if len(readonly_user_ids) > 0:
+                    readonly_auth_db = AuthorizationDB(
+                        creator=user_id,
+                        dataset_id=PydanticObjectId(dataset_id),
+                        role=RoleType.VIEWER,
+                        group_ids=[PydanticObjectId(group_id)],
+                        user_ids=readonly_user_ids,
+                    )
+                    await readonly_auth_db.insert()
+                    await index_dataset(
+                        es, DatasetOut(**dataset.dict()), readonly_auth_db.user_ids
+                    )
+                    await index_dataset_files(es, str(dataset_id))
+                if len(user_ids) > 0:
+                    auth_db = AuthorizationDB(
+                        creator=user_id,
+                        dataset_id=PydanticObjectId(dataset_id),
+                        role=role,
+                        group_ids=[PydanticObjectId(group_id)],
+                        user_ids=user_ids,
+                    )
+                    # if there are read only users add them with the role of viewer
+                    await auth_db.insert()
+                    await index_dataset(
+                        es, DatasetOut(**dataset.dict()), auth_db.user_ids
+                    )
+                    await index_dataset_files(es, str(dataset_id))
                 return auth_db.dict()
         else:
             raise HTTPException(status_code=404, detail=f"Group {group_id} not found")
@@ -303,6 +314,7 @@ async def set_dataset_user_role(
                     auth_db.user_ids.append(username)
                     await auth_db.save()
                 await index_dataset(es, DatasetOut(**dataset.dict()), auth_db.user_ids)
+                await index_dataset_files(es, dataset_id, update=True)
                 return auth_db.dict()
             else:
                 # Create a new entry
@@ -314,6 +326,7 @@ async def set_dataset_user_role(
                 )
                 await auth_db.insert()
                 await index_dataset(es, DatasetOut(**dataset.dict()), [username])
+                await index_dataset_files(es, dataset_id)
                 return auth_db.dict()
         else:
             raise HTTPException(status_code=404, detail=f"User {username} not found")
@@ -351,6 +364,7 @@ async def remove_dataset_group_role(
                 await auth_db.save()
                 # Update elasticsearch index with new users
                 await index_dataset(es, DatasetOut(**dataset.dict()), auth_db.user_ids)
+                await index_dataset_files(es, str(dataset_id), update=True)
                 return auth_db.dict()
         else:
             raise HTTPException(status_code=404, detail=f"Group {group_id} not found")
@@ -387,6 +401,7 @@ async def remove_dataset_user_role(
                 await auth_db.save()
                 # Update elasticsearch index with updated users
                 await index_dataset(es, DatasetOut(**dataset.dict()), auth_db.user_ids)
+                await index_dataset_files(es, dataset_id, update=True)
                 return auth_db.dict()
         else:
             raise HTTPException(status_code=404, detail=f"User {username} not found")

backend/app/routers/groups.py

@@ -1,13 +1,16 @@
 from datetime import datetime
 from typing import Optional
 
+from app import dependencies
 from app.deps.authorization_deps import AuthorizationDB, GroupAuthorization
 from app.keycloak_auth import get_current_user, get_user
 from app.models.authorization import RoleType
+from app.models.datasets import DatasetDB, DatasetOut
 from app.models.groups import GroupBase, GroupDB, GroupIn, GroupOut, Member
 from app.models.pages import Paged, _construct_page_metadata, _get_page_query
 from app.models.users import UserDB, UserOut
 from app.routers.authentication import get_admin, get_admin_mode
+from app.search.index import index_dataset, index_dataset_files
 from beanie import PydanticObjectId
 from beanie.operators import Or, Push, RegEx
 from bson.objectid import ObjectId
@@ -218,6 +221,7 @@ async def add_member(
     group_id: str,
     username: str,
     role: Optional[str] = None,
+    es=Depends(dependencies.get_elasticsearchclient),
     allow: bool = Depends(GroupAuthorization("editor")),
 ):
     """Add a new user to a group."""
@@ -245,6 +249,20 @@ async def add_member(
                 ).update(
                     Push({AuthorizationDB.user_ids: username}),
                 )
+                # index the datasets in the group
+                group_authorizations = await AuthorizationDB.find(
+                    AuthorizationDB.group_ids == ObjectId(group_id)
+                ).to_list()
+                for auth in group_authorizations:
+                    if (
+                        dataset := await DatasetDB.get(
+                            PydanticObjectId(auth.dataset_id)
+                        )
+                    ) is not None:
+                        await index_dataset(
+                            es, DatasetOut(**dataset.dict()), auth.user_ids
+                        )
+                        await index_dataset_files(es, str(auth.dataset_id), update=True)
             return group.dict()
         raise HTTPException(status_code=404, detail=f"Group {group_id} not found")
     raise HTTPException(status_code=404, detail=f"User {username} not found")
@@ -254,6 +272,7 @@ async def add_member(
 async def remove_member(
     group_id: str,
     username: str,
+    es=Depends(dependencies.get_elasticsearchclient),
     allow: bool = Depends(GroupAuthorization("editor")),
 ):
     """Remove a user from a group."""
@@ -278,6 +297,16 @@ async def remove_member(
         # Update group itself
         group.users.remove(found_user)
         await group.replace()
+        # index the datasets in the group
+        group_authorizations = await AuthorizationDB.find(
+            AuthorizationDB.group_ids == ObjectId(group_id)
+        ).to_list()
+        for auth in group_authorizations:
+            if (
+                dataset := await DatasetDB.get(PydanticObjectId(auth.dataset_id))
+            ) is not None:
+                await index_dataset(es, DatasetOut(**dataset.dict()), auth.user_ids)
+                await index_dataset_files(es, str(auth.dataset_id), update=True)
 
         return group.dict()
     raise HTTPException(status_code=404, detail=f"Group {group_id} not found")

backend/app/search/index.py

@@ -114,6 +114,15 @@ async def index_file(
         insert_record(es, settings.elasticsearch_index, doc, file.id)
 
 
+async def index_dataset_files(es: Elasticsearch, dataset_id: str, update: bool = False):
+    query = [
+        FileDB.dataset_id == ObjectId(dataset_id),
+    ]
+    files = await FileDB.find(*query).to_list()
+    for file in files:
+        await index_file(es, FileOut(**file.dict()), update=update)
+
+
 async def index_folder(
     es: Elasticsearch,
     folder: FolderOut,