ref migration, add payload to exports

Author: githubering182

backend-app/archive/services.py

@@ -40,7 +40,7 @@ def _make_archive(project: Project, request: Request) -> tuple[dict[str, Any], i
         attributes = project.attribute_set \
             .filter(id__in=request_query.get("attr", [])) \
             .order_by("level__order", "id") \
-            .values("name") \
+            .only("name") \
             .values_list("name", flat=True)
 
         only_new = request_query.get("only_new", False) is True

backend-app/attribute/serializers.py

@@ -35,6 +35,6 @@ class Meta:
 
     def get_attributes(self, instance: AttributeGroup) -> tuple[tuple[int, int, str]]:
         return tuple(
-            (attr.id, attr.level.order, attr.name)
+            (attr.id, attr.level.order, attr.name, attr.payload)
             for attr in instance.attribute.all()
         )

backend-app/file/services.py

@@ -137,7 +137,7 @@ def _get_files(
                     "attributegroup_set__attribute",
                     queryset=Attribute.objects.select_related("level")
                     .order_by("level__order", "level_id")
-                    .only("id", "name", "level__order")
+                    .only("id", "name", "level__order", "payload")
                 )
             ) \
             .annotate(
@@ -159,6 +159,7 @@ def _get_files(
                 .filter(attribute__in=attribute_query) \
                 .annotate(count=Count("uid")) \
                 .filter(count=len(attribute_query)) \
+                .only("uid") \
                 .values("uid")
             files = files.filter(attributegroup__in=Subquery(sub_query))
         else: files = files.distinct()
@@ -259,8 +260,9 @@ class StatsServices:
         "attribute__id",
         "attribute__name",
         "attribute__parent",
+        "attribute__payload",
         "attribute__attributegroup__file__file_type",
-        "attribute__attributegroup__file__status"
+        "attribute__attributegroup__file__status",
     )
 
     _USER_QUERY_VALUES: tuple[str, ...] = (
@@ -327,6 +329,7 @@ def from_attribute(cls, project_id: int) -> tuple[list[dict[str, Any]], int]:
                 "attribute__name": "No attribute",
                 "attribute__parent": None,
                 "attribute__id": "no-id",
+                "attribute__payload": "",
                 "attribute__attributegroup__file__file_type": entry["file_type"],
                 "attribute__attributegroup__file__status": entry["status"],
                 "count": entry["file_count"]

frontend-app/src/components/FilesDownload/index.jsx

@@ -51,7 +51,7 @@ export default function FilesDownload({ pathID, attributes }) {
       data: TYPE_FILTER,
     },
     {
-      prettyName: "Date Filter:",
+      prettyName: "Upload date Filter:",
       name: "date",
       type: "date",
     }

frontend-app/src/components/forms/AttributesForm/index.jsx

@@ -72,7 +72,7 @@ export default function AttributesForm({
     }
   };
 
-  const handleLevelDelete = (index, orig, _id) => {
+  const handleLevelDelete = (index, orig) => {
     if (orig) return setAcceptDelete(index);
     if (index === 0) deleteForm(formId);
     else {

nginx/nginx.conf.template

@@ -12,7 +12,7 @@ server {
     proxy_buffering off;
     client_max_body_size 5000M;
     proxy_connect_timeout 60s;
-    proxy_read_timeout 60s;
+    proxy_read_timeout 180s;
 
     add_header 'Access-Control-Allow-Origin' $http_origin always;
     add_header 'Access-Control-Allow-Credentials' 'true' always;

scripts/project_migrate.py

@@ -3,6 +3,69 @@
 from attribute.models import Attribute, Level
 
 
+def migrate_third_attrs(target: int, dest: int, from_order: int = 1):
+    attr_data = File.objects \
+        .filter(
+            project_id=dest,
+            rebound_project=target,
+            attributegroup__attribute__project_id=target,
+            attributegroup__attribute__level__order__gte=from_order
+        ) \
+        .values_list(
+            "attributegroup__uid",
+            "attributegroup__attribute__id",
+            "attributegroup__attribute__name",
+            "attributegroup__attribute__project_id",
+            "attributegroup__attribute__level__order"
+        ) \
+        .distinct()
+    print(f"[1/4] Found {attr_data.count()} attrs to migrate")
+
+    rebound = []
+    created = 0
+    failed = []
+
+    cache = {}
+
+    for ag_id, a_id, a_name, p_id, l_order in attr_data:
+        ag_id = str(ag_id)
+
+        if p_id != target: continue
+
+        if (cached := cache.get((a_id, a_name))):
+            print(f"[2/4] Cached {a_name}")
+            rebound.append((cached, a_id, ag_id,))
+            continue
+
+        try:
+            map_id = Attribute.objects.filter(project_id=dest, name=a_name).id
+            rebound.append((map_id, a_id, ag_id))
+            cache[(a_id, a_name)] = map_id
+            print(f"[2/4] Found {a_name} in dest")
+        except:
+            levels = Level.objects.filter(project_id=dest, order=l_order)
+            if levels.count() == 1:
+                map_id = levels.first().attribute_set.create(name=a_name, project_id=dest).id
+                rebound.append((map_id, a_id, ag_id))
+                created += 1
+                cache[(a_id, a_name)] = map_id
+                print(f"[2/4] Created {a_name} in dest")
+            else: failed.append((a_id, a_name))
+
+    print(f"[3/4] Got {len(rebound)} attributes with {created} created")
+
+    with connection.cursor() as cursor: cursor.executemany(
+        """
+            update attribute_group_attribute
+            set attribute_id = %s
+            where attribute_id = %s and attributegroup_id = %s;
+        """,
+        rebound
+    )
+
+    print(f"[4/4] Migrate done with {len(failed)} failed")
+
+
 def migrate_files(
     target_project: int,
     dest_project: int,
@@ -46,6 +109,7 @@ def get_ids(sep: str, path_from: str, path_to: str) -> tuple[set[str], set[str]]
         set([row.split(sep)[0] for row in data_to.split("\n")[1:] if row])
     )
 
+
 def main(target: tuple[int, str], dest: tuple[int, str], sep: str):
     target_id, target_file = target
     dest_id, dest_file = dest
@@ -83,3 +147,6 @@ def main(target: tuple[int, str], dest: tuple[int, str], sep: str):
             .filter(project_id=target_id, attributegroup__attribute__payload__in=target_set) \
             .update(rebound_project=target_id, project_id=dest_id)
         print(f"[4/4] Rebound {res} total files to the new projects")
+
+
+        # migrate_third_attrs(target_id, dest_id, 1)