changes for admin dashbaord extention (#43)

* Changes submodule url to ssh

* Adds alembic file for comments & user/org management

* submodule update & new query

* Adds get comments query

* Adds new endpoints

* Adds new endpoints

* enable adding manual rlas with source id

* Adds logic to remove annotator labels

* Adds logic to collect first heuristic

* adds stats and states to crowd labeling

* adds confidence to manually labeled data

* crowd labeling for ner

* adds update progress on payload

* adds role mgmt to the api

* Adds lock option and removes unused table

* Adds link refresh on outdated data

* Changes faulty alembic version

* Adds comment preparations

* Submodule change and role management

* Merge issue resolve

* Adds notification time for link chnages after the commit

* Import/Export of projects addition

* resolves pr comments

* Submodule change

Co-authored-by: Johannes Hötter <[email protected]>

Author: JWittmeyer

.gitmodules

@@ -1,6 +1,6 @@
 [submodule "submodules/model"]
 	path = submodules/model
-	url = https://github.com/code-kern-ai/refinery-submodule-model.git
+	url = git@github.com:code-kern-ai/refinery-submodule-model.git
 [submodule "submodules/s3"]
 	path = submodules/s3
-	url = https://github.com/code-kern-ai/refinery-submodule-s3.git
+	url = git@github.com:code-kern-ai/refinery-submodule-s3.git

alembic/versions/9618924f9679_adds_comment_link_tables_some_org_.py

@@ -0,0 +1,125 @@
+"""Adds comment & link tables & some org management
+
+Revision ID: 9618924f9679
+Revises: 5b3a4deea1c4
+Create Date: 2022-09-15 07:15:02.934928
+
+"""
+from alembic import op
+import sqlalchemy as sa
+from sqlalchemy.dialects import postgresql
+
+# revision identifiers, used by Alembic.
+revision = "9618924f9679"
+down_revision = "5b3a4deea1c4"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.create_table(
+        "comment",
+        sa.Column("id", postgresql.UUID(as_uuid=True), nullable=False),
+        sa.Column("project_id", postgresql.UUID(as_uuid=True), nullable=True),
+        sa.Column("xfkey", postgresql.UUID(as_uuid=True), nullable=True),
+        sa.Column("xftype", sa.String(), nullable=True),
+        sa.Column("order_key", sa.Integer(), autoincrement=True, nullable=True),
+        sa.Column("comment", sa.String(), nullable=True),
+        sa.Column("is_markdown", sa.Boolean(), nullable=True),
+        sa.Column("is_private", sa.Boolean(), nullable=True),
+        sa.Column("created_by", postgresql.UUID(as_uuid=True), nullable=True),
+        sa.Column("created_at", sa.DateTime(), nullable=True),
+        sa.ForeignKeyConstraint(
+            ["created_by"],
+            ["user.id"],
+        ),
+        sa.ForeignKeyConstraint(["project_id"], ["project.id"], ondelete="CASCADE"),
+        sa.PrimaryKeyConstraint("id"),
+    )
+    op.create_index(
+        op.f("ix_comment_created_by"), "comment", ["created_by"], unique=False
+    )
+    op.create_index(
+        op.f("ix_comment_project_id"), "comment", ["project_id"], unique=False
+    )
+    op.create_index(op.f("ix_comment_xfkey"), "comment", ["xfkey"], unique=False)
+    op.create_index(op.f("ix_comment_xftype"), "comment", ["xftype"], unique=False)
+    op.create_table(
+        "labeling_access_link",
+        sa.Column("id", postgresql.UUID(as_uuid=True), nullable=False),
+        sa.Column("project_id", postgresql.UUID(as_uuid=True), nullable=True),
+        sa.Column("link", sa.String(), nullable=True),
+        sa.Column("data_slice_id", postgresql.UUID(as_uuid=True), nullable=True),
+        sa.Column("heuristic_id", postgresql.UUID(as_uuid=True), nullable=True),
+        sa.Column("link_type", sa.String(), nullable=True),
+        sa.Column("created_at", sa.DateTime(), nullable=True),
+        sa.Column("created_by", postgresql.UUID(as_uuid=True), nullable=True),
+        sa.Column("is_locked", sa.Boolean(), nullable=True),
+        sa.Column("changed_at", sa.DateTime(), nullable=True),
+        sa.ForeignKeyConstraint(["project_id"], ["project.id"], ondelete="CASCADE"),
+        sa.ForeignKeyConstraint(["created_by"], ["user.id"], ondelete="CASCADE"),
+        sa.ForeignKeyConstraint(
+            ["data_slice_id"], ["data_slice.id"], ondelete="CASCADE"
+        ),
+        sa.ForeignKeyConstraint(
+            ["heuristic_id"], ["information_source.id"], ondelete="CASCADE"
+        ),
+        sa.PrimaryKeyConstraint("id"),
+    )
+    op.create_index(
+        op.f("ix_labeling_access_link_project_id"),
+        "labeling_access_link",
+        ["project_id"],
+        unique=False,
+    )
+    op.create_index(
+        op.f("ix_labeling_access_link_created_by"),
+        "labeling_access_link",
+        ["created_by"],
+        unique=False,
+    )
+    op.create_index(
+        op.f("ix_labeling_access_link_data_slice_id"),
+        "labeling_access_link",
+        ["data_slice_id"],
+        unique=False,
+    )
+    op.create_index(
+        op.f("ix_labeling_access_link_heuristic_id"),
+        "labeling_access_link",
+        ["heuristic_id"],
+        unique=False,
+    )
+    op.add_column("organization", sa.Column("started_at", sa.DateTime(), nullable=True))
+    op.add_column("organization", sa.Column("is_paying", sa.Boolean(), nullable=True))
+    op.add_column("organization", sa.Column("created_at", sa.DateTime(), nullable=True))
+    op.add_column("user", sa.Column("role", sa.String(), nullable=True))
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_column("user", "role")
+    op.drop_column("organization", "created_at")
+    op.drop_column("organization", "is_paying")
+    op.drop_column("organization", "started_at")
+    op.drop_index(
+        op.f("ix_labeling_access_link_project_id"), table_name="labeling_access_link"
+    )
+    op.drop_index(
+        op.f("ix_labeling_access_link_heuristic_id"), table_name="labeling_access_link"
+    )
+    op.drop_index(
+        op.f("ix_labeling_access_link_data_slice_id"), table_name="labeling_access_link"
+    )
+    op.drop_index(
+        op.f("ix_labeling_access_link_created_by"), table_name="labeling_access_link"
+    )
+    op.drop_table("labeling_access_link")
+    op.drop_index(op.f("ix_comment_xftype"), table_name="comment")
+    op.drop_index(op.f("ix_comment_xfkey"), table_name="comment")
+    op.drop_index(op.f("ix_comment_project_id"), table_name="comment")
+    op.drop_index(op.f("ix_comment_created_by"), table_name="comment")
+    op.drop_table("comment")
+    # ### end Alembic commands ###

api/project.py

@@ -5,6 +5,7 @@
 from controller.auth import manager as auth_manager
 from controller.project import manager as project_manager
 from controller.attribute import manager as attribute_manager
+from submodules.model import exceptions
 
 
 logging.basicConfig(level=logging.DEBUG)
@@ -14,7 +15,14 @@ class ProjectDetails(HTTPEndpoint):
     def get(self, request) -> JSONResponse:
         project_id = request.path_params["project_id"]
         user_id = request.query_params["user_id"]
-        auth_manager.check_project_access_from_user_id(user_id, project_id)
+        try:
+            auth_manager.check_project_access_from_user_id(
+                user_id, project_id, from_api=True
+            )
+        except exceptions.EntityNotFoundException:
+            return JSONResponse({"error": "Could not find project"}, status_code=404)
+        except exceptions.AccessDeniedException:
+            return JSONResponse({"error": "Access denied"}, status_code=403)
         project = project_manager.get_project(project_id)
         attributes = attribute_manager.get_all_attributes(project_id)
         result = {

api/transfer.py

@@ -14,7 +14,7 @@
 from controller.transfer import association_transfer_manager
 from controller.auth import manager as auth
 
-from submodules.model import enums
+from submodules.model import enums, exceptions
 from util.notification import create_notification
 from submodules.model.enums import NotificationType
 from submodules.model.models import UploadTask
@@ -63,7 +63,14 @@ def get(self, request) -> JSONResponse:
         project_id = request.path_params["project_id"]
         user_id = request.query_params["user_id"]
         num_samples = request.query_params.get("num_samples")
-        auth_manager.check_project_access_from_user_id(user_id, project_id)
+        try:
+            auth_manager.check_project_access_from_user_id(
+                user_id, project_id, from_api=True
+            )
+        except exceptions.EntityNotFoundException:
+            return JSONResponse({"error": "Could not find project"}, status_code=404)
+        except exceptions.AccessDeniedException:
+            return JSONResponse({"error": "Access denied"}, status_code=403)
         result = transfer_manager.export_records(project_id, num_samples)
         return JSONResponse(result)
 
@@ -73,7 +80,14 @@ def get(self, request) -> JSONResponse:
         project_id = request.path_params["project_id"]
         list_id = request.path_params["knowledge_base_id"]
         user_id = request.query_params["user_id"]
-        auth_manager.check_project_access_from_user_id(user_id, project_id)
+        try:
+            auth_manager.check_project_access_from_user_id(
+                user_id, project_id, from_api=True
+            )
+        except exceptions.EntityNotFoundException:
+            return JSONResponse({"error": "Could not find project"}, status_code=404)
+        except exceptions.AccessDeniedException:
+            return JSONResponse({"error": "Access denied"}, status_code=403)
         result = transfer_manager.export_knowledge_base(project_id, list_id)
         return JSONResponse(result)
 
@@ -85,7 +99,14 @@ async def post(self, request) -> JSONResponse:
         request_body = await request.json()
 
         user_id = request_body["user_id"]
-        auth_manager.check_project_access_from_user_id(user_id, project_id)
+        try:
+            auth_manager.check_project_access_from_user_id(
+                user_id, project_id, from_api=True
+            )
+        except exceptions.EntityNotFoundException:
+            return JSONResponse({"error": "Could not find project"}, status_code=404)
+        except exceptions.AccessDeniedException:
+            return JSONResponse({"error": "Access denied"}, status_code=403)
         file_name = request_body["file_name"]
         file_type = request_body["file_type"]
         file_import_options = request_body.get("file_import_options")
@@ -121,7 +142,14 @@ async def post(self, request) -> JSONResponse:
         project_id = request.path_params["project_id"]
         request_body = await request.json()
         user_id = request_body["user_id"]
-        auth_manager.check_project_access_from_user_id(user_id, project_id)
+        try:
+            auth_manager.check_project_access_from_user_id(
+                user_id, project_id, from_api=True
+            )
+        except exceptions.EntityNotFoundException:
+            return JSONResponse({"error": "Could not find project"}, status_code=404)
+        except exceptions.AccessDeniedException:
+            return JSONResponse({"error": "Access denied"}, status_code=403)
         new_associations_added = association_transfer_manager.import_associations(
             project_id,
             user_id,
@@ -140,7 +168,14 @@ def get(self, request) -> JSONResponse:
         project_id = request.path_params["project_id"]
         task_id = request.path_params["task_id"]
         user_id = request.query_params["user_id"]
-        auth_manager.check_project_access_from_user_id(user_id, project_id)
+        try:
+            auth_manager.check_project_access_from_user_id(
+                user_id, project_id, from_api=True
+            )
+        except exceptions.EntityNotFoundException:
+            return JSONResponse({"error": "Could not find project"}, status_code=404)
+        except exceptions.AccessDeniedException:
+            return JSONResponse({"error": "Access denied"}, status_code=403)
         task = upload_task_manager.get_upload_task(project_id, task_id)
         task_dict = {
             "id": str(task.id),

controller/auth/manager.py

@@ -8,8 +8,10 @@
 from controller.project import manager as project_manager
 from controller.user import manager as user_manager
 from controller.organization import manager as organization_manager
+from submodules.model import enums, exceptions
 from submodules.model.models import Organization, Project, User
 from controller.misc import manager as misc_manager
+import sqlalchemy
 
 
 def get_organization_id_by_info(info) -> Organization:
@@ -62,13 +64,22 @@ def check_admin_access(info) -> None:
         raise GraphQLError("Admin access required")
 
 
-def check_project_access_from_user_id(user_id: str, project_id: str) -> bool:
+def check_project_access_from_user_id(
+    user_id: str, project_id: str, from_api: bool = False
+) -> bool:
     organization_id: str = get_organization_by_user_id(user_id).id
-    project: Project = project_manager.get_project_with_orga_id(
-        organization_id, project_id
-    )
+    try:
+        project: Project = project_manager.get_project_with_orga_id(
+            organization_id, project_id
+        )
+    except sqlalchemy.exc.DataError:
+        raise exceptions.EntityNotFoundException("Project not found")
     if project is None:
-        raise GraphQLError("Project not found")
+        raise exceptions.EntityNotFoundException("Project not found")
+    if from_api:
+        user = user_manager.get_user(user_id)
+        if user.role != enums.UserRoles.ENGINEER.value:
+            raise exceptions.AccessDeniedException("Access denied")
     return True
 
 

controller/comment/__init__.py

@@ -0,0 +1,78 @@
+from typing import Any, Dict, List, Optional, Union
+from submodules.model import enums
+from submodules.model.models import Comment, User
+
+from submodules.model.business_objects import comments
+
+
+def has_comments(
+    xftype: str,
+    xfkey: Optional[str] = None,
+    project_id: Optional[str] = None,
+    group_by_xfkey: bool = False,
+) -> Union[bool, Dict[str, bool]]:
+
+    try:
+        xftype_parsed = enums.CommentCategory[xftype.upper()]
+    except KeyError:
+        raise ValueError(f"Invalid comment category: {xftype}")
+
+    return comments.has_comments(xftype_parsed, xfkey, project_id, group_by_xfkey)
+
+
+def get_comments(
+    xftype: str,
+    user_id: str,
+    xfkey: Optional[str] = None,
+    project_id: Optional[str] = None,
+) -> str:
+
+    try:
+        xftype_parsed = enums.CommentCategory[xftype.upper()]
+    except KeyError:
+        raise ValueError(f"Invalid comment category: {xftype}")
+
+    return comments.get_by_all_by_category(
+        xftype_parsed, user_id, xfkey, project_id, True
+    )
+
+
+def create_comment(
+    xfkey: str,
+    xftype: str,
+    comment: str,
+    user_id: str,
+    project_id: Optional[str] = None,
+    is_private: Optional[bool] = None,
+) -> Comment:
+    try:
+        xftype = enums.CommentCategory[xftype.upper()].value
+    except KeyError:
+        raise ValueError(f"Invalid comment type: {xftype}")
+    comments.create(
+        xfkey,
+        xftype,
+        comment,
+        user_id,
+        project_id,
+        None,
+        None,
+        is_private,
+        None,
+        with_commit=True,
+    )
+
+
+def update_comment(
+    comment_id: str,
+    user: User,
+    changes: Dict[str, Any],
+) -> Comment:
+    item = comments.get(comment_id)
+
+    if not item:
+        raise ValueError(f"Can't find comment")
+
+    if user.role != enums.UserRoles.ENGINEER.value and user.id != item.created_by:
+        raise ValueError(f"Can't update comment")
+    comments.change(item, changes, with_commit=True)