feat: add support for filtering project and group listing by direct membership (#394)

Add support to filter the project and group list endpoints by using the query parameter `direct_member`.
When used, the group or project list will contain only items the current user is a direct member of.

Author: andre-code

components/renku_data_services/authz/authz.py

@@ -358,6 +358,31 @@ async def resources_with_permission(
                 ids.append(response.resource_object_id)
         return ids
 
+    async def resources_with_direct_membership(
+        self, user: base_models.APIUser, resource_type: ResourceType
+    ) -> list[str]:
+        """Get all the resource IDs (for a specific resource kind) that a specific user is a direct member of."""
+        resource_ids: list[str] = []
+        if user.id is None:
+            return resource_ids
+
+        rel_filter = RelationshipFilter(
+            resource_type=resource_type.value,
+            optional_subject_filter=SubjectFilter(subject_type=ResourceType.user.value, optional_subject_id=user.id),
+        )
+
+        responses: AsyncIterable[ReadRelationshipsResponse] = self.client.ReadRelationships(
+            ReadRelationshipsRequest(
+                consistency=Consistency(fully_consistent=True),
+                relationship_filter=rel_filter,
+            )
+        )
+
+        async for response in responses:
+            resource_ids.append(response.relationship.resource.object_id)
+
+        return resource_ids
+
     @_is_allowed(Scope.READ)  # The scope on the resource that allows the user to perform this check in the first place
     async def users_with_permission(
         self,

components/renku_data_services/namespace/api.spec.yaml

@@ -19,7 +19,7 @@ paths:
           style: form
           explode: true
           schema:
-            $ref: "#/components/schemas/PaginationRequest"
+            $ref: "#/components/schemas/GroupsGetQuery"
       responses:
         "200":
           description: List of groups
@@ -525,6 +525,15 @@ components:
           minimum_role:
             description: A minimum role to filter results by.
             $ref: "#/components/schemas/GroupRole"
+    GroupsGetQuery:
+      description: Query params for namespace get request
+      allOf:
+      - $ref: "#/components/schemas/PaginationRequest"
+      - properties:
+          direct_member:
+            description: A flag to filter groups where the user is a direct member.
+            type: boolean
+            default: false
     PaginationRequest:
       type: object
       additionalProperties: false

components/renku_data_services/namespace/apispec.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  api.spec.yaml
-#   timestamp: 2024-08-30T10:14:05+00:00
+#   timestamp: 2024-10-03T13:53:07+00:00
 
 from __future__ import annotations
 
@@ -83,10 +83,6 @@ class ErrorResponse(BaseAPISpec):
     error: Error
 
 
-class GroupsGetParametersQuery(BaseAPISpec):
-    params: Optional[PaginationRequest] = None
-
-
 class GroupResponse(BaseAPISpec):
     id: str = Field(
         ...,
@@ -263,6 +259,16 @@ class NamespaceGetQuery(PaginationRequest):
     )
 
 
+class GroupsGetQuery(PaginationRequest):
+    direct_member: bool = Field(
+        False, description="A flag to filter groups where the user is a direct member."
+    )
+
+
+class GroupsGetParametersQuery(BaseAPISpec):
+    params: Optional[GroupsGetQuery] = None
+
+
 class NamespacesGetParametersQuery(BaseAPISpec):
     params: Optional[NamespaceGetQuery] = None
 

components/renku_data_services/namespace/blueprints.py

@@ -29,12 +29,14 @@ def get_all(self) -> BlueprintFactoryResponse:
         """List all groups."""
 
         @authenticate(self.authenticator)
-        @validate_query(query=apispec.PaginationRequest)
+        @validate_query(query=apispec.GroupsGetQuery)
         @paginate
         async def _get_all(
-            _: Request, user: base_models.APIUser, pagination: PaginationRequest, query: apispec.PaginationRequest
+            _: Request, user: base_models.APIUser, pagination: PaginationRequest, query: apispec.GroupsGetQuery
         ) -> tuple[list[dict], int]:
-            groups, rec_count = await self.group_repo.get_groups(user=user, pagination=pagination)
+            groups, rec_count = await self.group_repo.get_groups(
+                user=user, pagination=pagination, direct_member=query.direct_member
+            )
             return (
                 validate_and_dump(apispec.GroupResponseList, groups),
                 rec_count,

components/renku_data_services/namespace/db.py

@@ -74,14 +74,18 @@ async def generate_user_namespaces(self, *, session: AsyncSession | None = None)
         return output
 
     async def get_groups(
-        self,
-        user: base_models.APIUser,
-        pagination: PaginationRequest,
+        self, user: base_models.APIUser, pagination: PaginationRequest, direct_member: bool = False
     ) -> tuple[list[models.Group], int]:
         """Get all groups from the database."""
+        if direct_member:
+            group_ids = await self.authz.resources_with_direct_membership(user, ResourceType.group)
+
         async with self.session_maker() as session, session.begin():
-            stmt = select(schemas.GroupORM).limit(pagination.per_page).offset(pagination.offset)
-            stmt = stmt.order_by(schemas.GroupORM.creation_date.asc(), schemas.GroupORM.id.asc(), schemas.GroupORM.name)
+            stmt = select(schemas.GroupORM)
+            if direct_member:
+                stmt = stmt.where(schemas.GroupORM.id.in_(group_ids))
+            stmt = stmt.limit(pagination.per_page).offset(pagination.offset)
+            stmt = stmt.order_by(schemas.GroupORM.id.desc())
             result = await session.execute(stmt)
             groups_orm = result.scalars().all()
 

components/renku_data_services/project/api.spec.yaml

@@ -283,6 +283,8 @@ components:
           $ref: "#/components/schemas/CreationDate"
         created_by:
           $ref: "#/components/schemas/UserId"
+        updated_at:
+          $ref: "#/components/schemas/UpdatedAt"
         repositories:
           $ref: "#/components/schemas/RepositoriesList"
         visibility:
@@ -385,6 +387,11 @@ components:
       type: string
       format: date-time
       example: "2023-11-01T17:32:28Z"
+    UpdatedAt:
+      description: The date and time the resource was updated (in UTC and ISO-8601 format)
+      type: string
+      format: date-time
+      example: "2023-11-01T17:32:28Z"
     Description:
       description: A description for the resource
       type: string
@@ -504,6 +511,10 @@ components:
             description: A namespace, used as a filter.
             type: string
             default: ""
+          direct_member:
+            description: A flag to filter projects where the user is a direct member.
+            type: boolean
+            default: false
     PaginationRequest:
       type: object
       additionalProperties: false

components/renku_data_services/project/apispec.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  api.spec.yaml
-#   timestamp: 2024-08-30T09:58:32+00:00
+#   timestamp: 2024-09-30T13:50:44+00:00
 
 from __future__ import annotations
 
@@ -105,6 +105,10 @@ class ProjectMemberResponse(BaseAPISpec):
 
 class ProjectGetQuery(PaginationRequest):
     namespace: str = Field("", description="A namespace, used as a filter.")
+    direct_member: bool = Field(
+        False,
+        description="A flag to filter projects where the user is a direct member.",
+    )
 
 
 class ProjectsGetParametersQuery(BaseAPISpec):
@@ -153,6 +157,11 @@ class Project(BaseAPISpec):
         example="f74a228b-1790-4276-af5f-25c2424e9b0c",
         pattern="^[A-Za-z0-9]{1}[A-Za-z0-9-]+$",
     )
+    updated_at: Optional[datetime] = Field(
+        None,
+        description="The date and time the resource was updated (in UTC and ISO-8601 format)",
+        example="2023-11-01T17:32:28Z",
+    )
     repositories: Optional[List[str]] = Field(
         None,
         description="A list of repositories",

components/renku_data_services/project/blueprints.py

@@ -46,7 +46,7 @@ async def _get_all(
             request: Request, user: base_models.APIUser, pagination: PaginationRequest, query: apispec.ProjectGetQuery
         ) -> tuple[list[dict[str, Any]], int]:
             projects, total_num = await self.project_repo.get_projects(
-                user=user, pagination=pagination, namespace=query.namespace
+                user=user, pagination=pagination, namespace=query.namespace, direct_member=query.direct_member
             )
             return [
                 dict(
@@ -56,6 +56,7 @@ async def _get_all(
                     slug=p.slug,
                     creation_date=p.creation_date.isoformat(),
                     created_by=p.created_by,
+                    updated_at=p.updated_at.isoformat() if p.updated_at else None,
                     repositories=p.repositories,
                     visibility=p.visibility.value,
                     description=p.description,

components/renku_data_services/project/db.py

@@ -9,6 +9,7 @@
 
 from sqlalchemy import Select, delete, func, select, update
 from sqlalchemy.ext.asyncio import AsyncSession
+from sqlalchemy.sql.functions import coalesce
 from ulid import ULID
 
 import renku_data_services.base_models as base_models
@@ -48,18 +49,29 @@ def __init__(
         self.authz = authz
 
     async def get_projects(
-        self, user: base_models.APIUser, pagination: PaginationRequest, namespace: str | None = None
+        self,
+        user: base_models.APIUser,
+        pagination: PaginationRequest,
+        namespace: str | None = None,
+        direct_member: bool = False,
     ) -> tuple[list[models.Project], int]:
         """Get all projects from the database."""
-        project_ids = await self.authz.resources_with_permission(user, user.id, ResourceType.project, Scope.READ)
+        project_ids = []
+        if direct_member:
+            project_ids = await self.authz.resources_with_direct_membership(user, ResourceType.project)
+        else:
+            project_ids = await self.authz.resources_with_permission(user, user.id, ResourceType.project, Scope.READ)
 
         async with self.session_maker() as session:
             stmt = select(schemas.ProjectORM)
             stmt = stmt.where(schemas.ProjectORM.id.in_(project_ids))
             if namespace:
                 stmt = _filter_by_namespace_slug(stmt, namespace)
+
+            stmt = stmt.order_by(coalesce(schemas.ProjectORM.updated_at, schemas.ProjectORM.creation_date).desc())
+
             stmt = stmt.limit(pagination.per_page).offset(pagination.offset)
-            stmt = stmt.order_by(schemas.ProjectORM.creation_date.desc())
+
             stmt_count = (
                 select(func.count()).select_from(schemas.ProjectORM).where(schemas.ProjectORM.id.in_(project_ids))
             )

test/bases/renku_data_services/data_api/test_groups.py

@@ -91,15 +91,15 @@ async def test_group_pagination(
     res2_json = res2.json
     assert len(res1_json) == 12
     assert len(res2_json) == 3
-    assert res1_json[0]["name"] == "group0"
-    assert res1_json[-1]["name"] == "group11"
-    assert res2_json[0]["name"] == "group12"
-    assert res2_json[-1]["name"] == "group14"
+    assert res1_json[0]["name"] == "group14"
+    assert res1_json[-1]["name"] == "group3"
+    assert res2_json[0]["name"] == "group2"
+    assert res2_json[-1]["name"] == "group0"
     _, res3 = await sanic_client.get("/api/data/groups", headers=admin_headers, params={"per_page": 20, "page": 1})
     res3_json = res3.json
     assert len(res3_json) == 15
-    assert res3_json[0]["name"] == "group0"
-    assert res3_json[-1]["name"] == "group14"
+    assert res3_json[0]["name"] == "group14"
+    assert res3_json[-1]["name"] == "group0"
 
 
 @pytest.mark.asyncio
@@ -461,3 +461,47 @@ async def test_get_group_anonymously(sanic_client, user_headers) -> None:
     member_2 = members[1]
     assert member_2["id"] == "member-1"
     assert member_2["role"] == "viewer"
+
+
+@pytest.mark.asyncio
+async def test_get_groups_with_direct_membership(sanic_client, user_headers, member_1_headers, member_1_user) -> None:
+    # Create a group
+    namespace = "my-group"
+    payload = {
+        "name": "Group",
+        "slug": namespace,
+    }
+    _, response = await sanic_client.post("/api/data/groups", headers=user_headers, json=payload)
+    assert response.status_code == 201, response.text
+    group_1 = response.json
+
+    # Create another group
+    namespace_2 = "my-second-group"
+    payload = {
+        "name": "Group 2",
+        "slug": namespace_2,
+    }
+    _, response = await sanic_client.post("/api/data/groups", headers=user_headers, json=payload)
+    assert response.status_code == 201, response.text
+    group_2 = response.json
+
+    # Add member_1 to Group 2
+    roles = [{"id": member_1_user.id, "role": "editor"}]
+    _, response = await sanic_client.patch(f"/api/data/groups/{namespace_2}/members", headers=user_headers, json=roles)
+    assert response.status_code == 200, response.text
+
+    # Get groups where member_1 has direct membership
+    parameters = {"direct_member": True}
+    _, response = await sanic_client.get("/api/data/groups", headers=member_1_headers, params=parameters)
+    assert response.status_code == 200, response.text
+    groups = response.json
+    assert len(groups) == 1
+    group_ids = {g["id"] for g in groups}
+    assert group_ids == {group_2["id"]}
+
+    # Check that both groups can be seen without the filter
+    _, response = await sanic_client.get("/api/data/groups", headers=member_1_headers)
+    groups = response.json
+    assert len(groups) == 2
+    group_ids = {g["id"] for g in groups}
+    assert group_ids == {group_1["id"], group_2["id"]}