add attribute diff stats tab

Author: githubering182

backend-app/attribute/models.py

@@ -10,14 +10,16 @@
     BooleanField,
     TextField,
     CASCADE,
-    IntegerField
+    IntegerField,
+    DateTimeField,
 )
 from tree_queries.models import TreeNode
 from uuid import uuid4
 
 
 class Attribute(TreeNode):
     name = CharField(max_length=255)
+    create_date = DateTimeField(auto_now_add=True)
 
     project = ForeignKey("project.Project", on_delete=DO_NOTHING)
     level = ForeignKey("attribute.Level", on_delete=DO_NOTHING)

backend-app/attribute/services.py

@@ -7,7 +7,7 @@
 )
 from rest_framework.views import Request, Response, APIView
 from django.db import transaction, IntegrityError
-from typing import Any
+from typing import Any, Optional
 from file.models import File
 from .models import Level, Attribute
 
@@ -105,3 +105,27 @@ def _check_intersection(self, item: Level | Attribute) -> bool:
         else: delete_ids.add(item.id)
 
         return bool(file_attributes.intersection(delete_ids))
+
+
+def _attribute_diff(
+    project_id: int,
+    diff_from: Optional[str]
+) -> tuple[list[tuple[int, str, str, int]], int]:
+    if not diff_from: return [], HTTP_200_OK
+
+    base_query = Attribute.objects \
+        .prefetch_related("level") \
+        .filter(project_id=project_id) \
+        .values_list("id", "name", "level__name")
+
+    attr_current = set(base_query)
+    attr_upto = set(base_query.filter(create_date__lt=diff_from))
+
+    into_response = lambda lst, tp: [(r[0], r[1], r[2], tp) for r in lst]
+
+    return (
+        into_response(attr_current - attr_upto, 1)
+        +
+        into_response(attr_upto - attr_current, 0),
+        HTTP_200_OK
+    )

backend-app/attribute/urls.py

@@ -1,9 +1,10 @@
 from django.urls import path
-from .views import LevelsViewSet, AttributesViewSet
+from .views import LevelsViewSet, AttributesViewSet, attribute_diff
 
 urlpatterns = (
     path("levels/", LevelsViewSet.as_view()),
     path("levels/<int:item_id>/", LevelsViewSet.as_view()),
     path("attributes/", AttributesViewSet.as_view()),
+    path("attributes/diff/<int:projectID>/", attribute_diff),
     path("attributes/<int:item_id>/", AttributesViewSet.as_view()),
 )

backend-app/attribute/views.py

@@ -1,7 +1,11 @@
+from rest_framework.views import Response, Request
 from rest_framework.permissions import IsAuthenticated
+from rest_framework.decorators import api_view, permission_classes
+from project.permissions import ProjectStatsPermission
+from typing import cast, Optional
 from .permissions import LevelPermission, AttributePermission
 from .serializers import Level, Attribute
-from .services import ViewMixIn
+from .services import ViewMixIn, _attribute_diff
 
 
 class LevelsViewSet(ViewMixIn):
@@ -12,3 +16,11 @@ class LevelsViewSet(ViewMixIn):
 class AttributesViewSet(ViewMixIn):
     permission_classes = (IsAuthenticated, AttributePermission)
     model = Attribute
+
+
+@api_view(("GET",))
+@permission_classes((IsAuthenticated, ProjectStatsPermission))
+def attribute_diff(request: Request, projectID: int) -> Response:
+    param = cast(Optional[str], request.query_params.get("diff_from"))
+    response, status = _attribute_diff(projectID, param)
+    return Response(response, status=status)

backend-app/file/export.py

@@ -80,7 +80,7 @@ def into_response(self) -> BytesIO:
         file = BytesIO()
 
         match self._type:
-            case "attribute":
+            case "attribute" | "diff":
                 headers = self.ATTRIBUTE_HEADERS
                 write = self._write_attribute
             case "user":

backend-app/file/services.py

@@ -4,7 +4,7 @@
     HTTP_201_CREATED,
     HTTP_200_OK,
     HTTP_202_ACCEPTED,
-    HTTP_404_NOT_FOUND
+    HTTP_404_NOT_FOUND,
 )
 from django.db.models import (
     Count,
@@ -24,7 +24,7 @@
 from attribute.models import Level, AttributeGroup, Attribute
 from user.models import CustomUser
 from json import loads
-from typing import Any
+from typing import Any, Optional
 from datetime import datetime as dt
 from io import BytesIO
 from .export import IMPLEMENTED, JSON, CSV, XLS
@@ -295,7 +295,105 @@ class StatsServices:
     )
 
     @classmethod
-    def from_attribute(cls, project_id: int) -> tuple[list[dict[str, Any]], int]:
+    def from_diff(
+        cls,
+        project_id: int,
+        diff_from: Optional[str]
+    ) -> tuple[list[dict[str, Any]], int]:
+        # TODO: refactor, unify, if this shows legit data
+        if not diff_from: return [], HTTP_200_OK
+
+        def helper(date: Optional[str]) -> list[tuple[Any, ...]]:
+            nonlocal project_id
+            query = """
+                select F.status, F.file_type, count(F.file_type) as count, L.order, L.name, A.id, A.name, A.parent_id, A.payload
+                from file as F
+                left join attribute_group AG on F.id = AG.file_id
+                left join attribute_group_attribute as AGA on AGA.attributegroup_id = AG.uid
+                left join attribute A on AGA.attribute_id = A.id
+                left join level as L on L.id = A.level_id
+                where F.project_id = {} {}
+                group by A.id, F.file_type, F.status, L.id;
+            """.format(project_id, f"and update_date < '{date}'" * bool(date))
+
+            with connection.cursor() as cur:
+                cur.execute(query, [])
+                return cur.fetchall()
+
+        stats_upto = helper(diff_from)
+        stats_current = helper(None)
+
+        intermediate = {}
+
+        for row in stats_current:
+            a_st, a_tp, cnt, order, l_name, a_id, a_name, a_par, a_payl = row
+
+            a_st = a_st or "v"
+            a_name = a_name or "no attribute"
+            a_tp = a_tp or "no type"
+            order = order or 0
+            l_name = l_name or "no level"
+            a_id = a_id or "no id"
+
+            target = intermediate.get(a_id)
+
+            if not target: intermediate[a_id] = {
+                "id": a_id,
+                "levelName": l_name,
+                "name": a_name,
+                "parent": a_par,
+                "payload": a_payl,
+                "order": order,
+                a_st: {a_tp: cnt}
+            }
+            elif target.get(a_st): target[a_st][a_tp] = target[a_st].get(a_tp, 0) + cnt
+            else: target[a_st] = {a_tp: cnt}
+
+        for row in stats_upto:
+            a_st, a_tp, cnt, order, l_name, a_id, a_name, a_par, a_payl = row
+
+            a_st = a_st or "v"
+            a_name = a_name or "no attribute"
+            a_tp = a_tp or "no type"
+            order = order or 0
+            l_name = l_name or "no level"
+            a_id = a_id or "no id"
+
+            target = intermediate.get(a_id)
+
+            if not target: intermediate[a_id] = {
+                "id": a_id,
+                "levelName": l_name,
+                "name": a_name,
+                "parent": a_par,
+                "payload": a_payl,
+                "order": order,
+                a_st: {a_tp: -cnt}
+            }
+            elif target.get(a_st): target[a_st][a_tp] = target[a_st].get(a_tp, 0) - cnt
+            else: target[a_st] = {a_tp: -cnt}
+
+        items = list(intermediate.items())
+        for key, row in items:
+            if not bool(
+                row.get("v", {}).get("image", 0)
+                | row.get("v", {}).get("video", 0)
+                | row.get("a", {}).get("image", 0)
+                | row.get("a", {}).get("video", 0)
+                | row.get("d", {}).get("image", 0)
+                | row.get("d", {}).get("video", 0)
+            ):
+                del intermediate[key]
+                continue
+
+            if parent := next((p for p in intermediate.values() if p["id"] == row["parent"]), None):
+                parent["children"] = parent.get("children", []) + [row]
+                del intermediate[key]
+
+        return sorted(intermediate.values(), key=lambda r: r["order"]), HTTP_200_OK
+
+    @classmethod
+    def from_attribute(cls, project_id: int, *args) -> tuple[list[dict[str, Any]], int]:
         stats: list[dict[str, Any]] = list(
             Level.objects
             .filter(project_id=project_id)
@@ -345,7 +443,7 @@ def from_attribute(cls, project_id: int) -> tuple[list[dict[str, Any]], int]:
         return cls._attribute_stat_adapt(stats), HTTP_200_OK
 
     @classmethod
-    def from_user(cls, project_id: int):
+    def from_user(cls, project_id: int, *args):
         stats: list[dict[str, Any]] = list(
             File.objects
                 .filter(project_id=project_id)
@@ -427,7 +525,8 @@ def form_export_file(query: dict[str, Any]) -> BytesIO:
     query_set = {"type", "project_id", "choice"}
     choice_map = {
         "attribute": StatsServices.from_attribute,
-        "user": StatsServices.from_user
+        "user": StatsServices.from_user,
+        "diff": StatsServices.from_diff
     }
     export_map = {"json": JSON, "csv": CSV, "xlsx": XLS}
 
@@ -438,11 +537,12 @@ def form_export_file(query: dict[str, Any]) -> BytesIO:
     choice = query["choice"]
     project_id = query["project_id"]
     file_type = query["type"]
+    diff_from = query.get("diff_from")
 
     assert file_type in IMPLEMENTED, f"{file_type} not implemented"
-    assert choice in set(choice_map), f"export for {choice} is not implemented"
+    assert (parser := choice_map.get(choice)), f"export for {choice} is not implemented"
 
-    stats, _ = choice_map[choice](project_id)
+    stats, _ = parser(project_id, diff_from)
     file = export_map[file_type](stats, choice)
 
     return file.into_response()

backend-app/file/urls.py

@@ -5,13 +5,15 @@
     get_user_stats,
     get_attribute_stats,
     export_stats,
-    get_duplicates
+    get_duplicates,
+    get_diff_stats
 )
 
 urlpatterns: tuple = (
     path("project/<int:projectID>/", FilesViewSet.as_view()),
     path("stats/attribute/<int:projectID>/", get_attribute_stats),
     path("stats/user/<int:projectID>/", get_user_stats),
+    path("stats/diff/<int:projectID>/", get_diff_stats),
     path("stats/export/", export_stats),
     path("duplicates/<str:fileID>/", get_duplicates),
     path("<str:fileID>/", FileViewSet.as_view()),

backend-app/file/views.py

@@ -4,6 +4,7 @@
 from rest_framework.permissions import IsAuthenticated
 from project.permissions import ProjectStatsPermission
 from django.http.response import FileResponse
+from typing import cast, Optional
 from .permissions import FilePermission
 from .services import (
     ViewSetServices,
@@ -53,9 +54,20 @@ def get_user_stats(_, projectID: int) -> Response:
     return Response(response, status=status)
 
 
+@api_view(("GET",))
+@permission_classes((IsAuthenticated, ProjectStatsPermission))
+def get_diff_stats(request: Request, projectID: int) -> Response:
+    param = cast(Optional[str], request.query_params.get("diff_from"))
+    response, status = StatsServices.from_diff(projectID, param)
+    return Response(response, status=status)
+
+
 @api_view(("GET",))
 @permission_classes((IsAuthenticated, ProjectStatsPermission))
 def export_stats(request: Request) -> Response | FileResponse:
+    response = form_export_file(request.query_params)
+    return FileResponse(response)
+
     try:
         response = form_export_file(request.query_params)
         return FileResponse(response)

docker-compose.dev.yml

@@ -6,6 +6,8 @@ services:
       DEBUG: ${DEBUG:-true}
     volumes:
       - ./backend-app:/app
+    command: >
+      sh -c "gunicorn --config './conf/gunicorn_conf.py' proj_back.wsgi:application"
 
   iss-storage:
     environment:

frontend-app/src/components/common/FileStats/component.test.js

@@ -22,7 +22,7 @@ test("files stats component test", async () => {
     </Provider>
   ));
 
-  screen.getByRole('table');
+  expect(screen.getAllByRole('table')).toHaveLength(2);
   expect(screen.queryByTestId('load-c')).toBeNull();
   expect(screen.getAllByRole("radio")[0].checked).toBeTruthy();
   expect(screen.getAllByRole("radio")[1].checked).toBeFalsy();