feat: speed up nice_list filter for analysis summary

Author: jstucke

src/storage/db_interface_common.py

@@ -214,7 +214,7 @@ def get_file_tree_path(self, uid: str, root_uid: str | None = None) -> list[list
         return self.get_file_tree_path_for_uid_list([uid], root_uid=root_uid).get(uid, [])
 
     def get_file_tree_path_for_uid_list(
-        self, uid_list: list[str], root_uid: str | None = None
+        self, uid_list: Iterable[str], root_uid: str | None = None
     ) -> dict[str, list[list[str]]]:
         """
         Generate all file paths for a list of UIDs `uid_list`. A path is a list of UIDs representing the path from root

src/storage/db_interface_frontend.py

@@ -3,7 +3,7 @@
 import os.path
 import re
 from pathlib import Path
-from typing import Any, NamedTuple, Optional
+from typing import Any, Iterable, NamedTuple, Optional
 
 from sqlalchemy import Column, Integer, cast, func, or_, select
 
@@ -75,24 +75,50 @@ def _get_hid_fo(self, uid: str, root_uid: str | None = None) -> str | None:
 
     # --- "nice list" ---
 
-    def get_data_for_nice_list(self, uid_list: list[str], root_uid: str | None) -> list[dict]:
+    def _get_some_path_for_uid_list(self, uid_list: Iterable[str], root_uid: str | None = None) -> dict[str, str]:
         with self.get_read_only_session() as session:
-            mime_dict = self._get_mime_types_for_uid_list(session, uid_list)
-            query = select(FileObjectEntry.uid, FileObjectEntry.size, FileObjectEntry.file_name).filter(
-                FileObjectEntry.uid.in_(uid_list)
+            query = (
+                select(VirtualFilePath.file_uid, VirtualFilePath.file_path)
+                .filter(VirtualFilePath.file_uid.in_(uid_list))
+                .distinct(VirtualFilePath.file_uid)  # only one path per file
+            )
+            if root_uid:  # if root_uid is set, only return paths contained in that FW
+                query = query.join(fw_files_table, VirtualFilePath.file_uid == fw_files_table.c.file_uid).filter(
+                    fw_files_table.c.root_uid == root_uid
+                )
+            return {uid: path for uid, path in session.execute(query)}  # noqa: C416  # dict() does not work here
+
+    def get_data_for_nice_list(
+        self, uid_list: Iterable[str], root_uid: str | None, include_vfp: bool = True
+    ) -> list[dict]:
+        with self.get_read_only_session() as session:
+            query = (
+                select(
+                    FileObjectEntry.uid,
+                    FileObjectEntry.size,
+                    FileObjectEntry.file_name,
+                    AnalysisEntry.result.op('->>')('mime'),
+                )
+                .filter(FileObjectEntry.uid.in_(uid_list))
+                .outerjoin(AnalysisEntry, AnalysisEntry.uid == FileObjectEntry.uid)
+                .filter(AnalysisEntry.plugin == 'file_type')
             )
-            file_tree_data = self.get_file_tree_path_for_uid_list(uid_list, root_uid=root_uid)
+            path_dict = self._get_some_path_for_uid_list(uid_list)
+
             nice_list_data = [
                 {
                     'uid': uid,
                     'size': size,
-                    'file_name': file_name,
-                    'mime-type': mime_dict.get(uid, 'file-type-plugin/not-run-yet'),
-                    'current_virtual_path': file_tree_data[uid],
+                    'file_name': path_dict.get(uid, file_name),
+                    'mime-type': mime or 'N/A',
                 }
-                for uid, size, file_name in session.execute(query)
+                for uid, size, file_name, mime in session.execute(query)
             ]
-            self._replace_uids_in_nice_list(nice_list_data, root_uid)
+            if include_vfp:
+                file_tree_data = self.get_file_tree_path_for_uid_list(uid_list, root_uid=root_uid)
+                for dict_ in nice_list_data:
+                    dict_['current_virtual_path'] = file_tree_data.get(dict_['uid'], [[dict_['file_name']]])
+                self._replace_uids_in_nice_list(nice_list_data, root_uid)
             return nice_list_data
 
     def _replace_uids_in_nice_list(self, nice_list_data: list[dict], root_uid: str):

src/test/common_helper.py

@@ -207,7 +207,7 @@ def uid_list_exists(self, uid_list):
     def all_uids_found_in_database(self, uid_list):
         return True
 
-    def get_data_for_nice_list(self, input_data, root_uid):
+    def get_data_for_nice_list(self, input_data, root_uid, include_vfp=None):
         return [NICE_LIST_DATA]
 
     @staticmethod

src/test/integration/storage/test_db_interface_frontend.py

@@ -90,6 +90,10 @@ def test_get_data_for_nice_list(frontend_db, backend_db):
     assert nice_list_data[0]['current_virtual_path'][0] == expected_hid, 'UID should be replaced with HID'
     assert nice_list_data[1]['current_virtual_path'][0] == f'{expected_hid} | /file/path'
 
+    nice_list_data = frontend_db.get_data_for_nice_list(uid_list, uid_list[0], include_vfp=False)
+    assert len(nice_list_data) == 2
+    assert sorted(nice_list_data[0].keys()) == ['file_name', 'mime-type', 'size', 'uid']
+
 
 def test_get_device_class_list(frontend_db, backend_db):
     insert_test_fw(backend_db, 'fw1', device_class='class1')

src/web_interface/components/ajax_routes.py

@@ -118,13 +118,16 @@ def ajax_get_hex_preview(self, uid: str, offset: int, length: int) -> str:
     def ajax_get_summary(self, uid, selected_analysis):
         with get_shared_session(self.db.frontend) as frontend_db:
             firmware = frontend_db.get_object(uid, analysis_filter=selected_analysis)
-            summary_of_included_files = frontend_db.get_summary(firmware, selected_analysis)
+            summary_of_included_files = frontend_db.get_summary(firmware, selected_analysis) or {}
+            all_uids = {uid for uid_list in summary_of_included_files.values() for uid in uid_list}
+            nice_list_data = self.db.frontend.get_data_for_nice_list(all_uids, uid, False)
             root_uid = uid if isinstance(firmware, Firmware) else frontend_db.get_root_uid(uid)
         return render_template(
             'summary.html',
             summary_of_included_files=summary_of_included_files,
             root_uid=root_uid,
             selected_analysis=selected_analysis,
+            nice_list_data=nice_list_data,
         )
 
     @roles_accepted(*PRIVILEGES['status'])

src/web_interface/components/jinja_filter.py

@@ -70,12 +70,23 @@ def _filter_replace_uid_with_hid_link(self, input_data, root_uid=None):
             content = content.replace(uid, f'<a style="text-reset" href="/analysis/{uid}/ro/{root_uid}">{hid}</a>')
         return content
 
-    def _filter_nice_uid_list(self, uids, root_uid=None, selected_analysis=None, filename_only=False):
+    def _filter_nice_uid_list(
+        self,
+        uids,
+        root_uid=None,
+        selected_analysis=None,
+        filename_only=False,
+        nice_list_data: list[dict] | None = None,
+    ):
         root_uid = none_to_none(root_uid)
         if not is_list_of_uids(uids):
             return uids
 
-        analyzed_uids = self.db.frontend.get_data_for_nice_list(uids, root_uid)
+        if nice_list_data:
+            # if multiple "nice lists" are rendered at once, nice_list_data should be prefetched for all files
+            analyzed_uids = [d for d in nice_list_data if d['uid'] in uids]
+        else:
+            analyzed_uids = self.db.frontend.get_data_for_nice_list(uids, root_uid, include_vfp=not filename_only)
         number_of_unanalyzed_files = len(uids) - len(analyzed_uids)
         first_item = analyzed_uids.pop(0)
 

src/web_interface/templates/generic_view/file_object_macros.html

@@ -1,7 +1,7 @@
 {% macro file_information_span(file_object, root_uid, selected_analysis=None, filename_only=False) %}
     {% set selected_analysis_url = selected_analysis + '/' if selected_analysis else '' %}
     <a href="/analysis/{{ file_object.uid }}/{{ selected_analysis_url }}ro/{{ root_uid }}">
-        {% if filename_only %}
+        {% if filename_only or "current_virtual_path" not in file_object %}
             <h6>{{ file_object.file_name | safe }}</h6>
         {% else %}
             <h6>{{ file_object.current_virtual_path[0] | safe }}</h6>

src/web_interface/templates/summary.html

@@ -17,7 +17,7 @@
             <tr>
                 <td>{{ item | wordwrap(width=60, break_long_words=True) }}</td>
                 <td>
-                    {{ summary_of_included_files[item] | nice_uid_list(root_uid=root_uid, selected_analysis=selected_analysis) | safe }}
+                    {{ summary_of_included_files[item] | nice_uid_list(root_uid=root_uid, selected_analysis=selected_analysis, nice_list_data=nice_list_data) | safe }}
                 </td>
             </tr>
         {% endfor %}