feat: improve performance of "Delete Missing Entries" (TagStudioDev#696)

* feat: Delete all unlinked entries at once (TagStudioDev#617)

This technically removes the usefulness progress indicator, but brief
testing on my end had it delete ~8000 entries in less time than it took
me to see if the progress indicator was properly indeterminate or not

* fix(fmt): remove unused import

* fix: wasn't able to delete more than 32766 entries

* chose: ruff check --fix

* fix: address review comment

---------

Co-authored-by: Tobias Berger <[email protected]>

Author: Computerdores

tagstudio/src/core/library/alchemy/enums.py

@@ -5,6 +5,8 @@
 from src.core.query_lang import AST as Query  # noqa: N811
 from src.core.query_lang import Constraint, ConstraintType, Parser
 
+MAX_SQL_VARIABLES = 32766  # 32766 is the max sql bind parameter count as defined here: https://github.com/sqlite/sqlite/blob/master/src/sqliteLimit.h#L140
+
 
 class TagColor(enum.IntEnum):
     DEFAULT = 1

tagstudio/src/core/library/alchemy/library.py

@@ -54,7 +54,7 @@
 )
 from ...enums import LibraryPrefs
 from .db import make_tables
-from .enums import FieldTypeEnum, FilterState, SortingModeEnum, TagColor
+from .enums import MAX_SQL_VARIABLES, FieldTypeEnum, FilterState, SortingModeEnum, TagColor
 from .fields import (
     BaseField,
     DatetimeField,
@@ -546,7 +546,11 @@ def add_entries(self, items: list[Entry]) -> list[int]:
     def remove_entries(self, entry_ids: list[int]) -> None:
         """Remove Entry items matching supplied IDs from the Library."""
         with Session(self.engine) as session:
-            session.query(Entry).where(Entry.id.in_(entry_ids)).delete()
+            for sub_list in [
+                entry_ids[i : i + MAX_SQL_VARIABLES]
+                for i in range(0, len(entry_ids), MAX_SQL_VARIABLES)
+            ]:
+                session.query(Entry).where(Entry.id.in_(sub_list)).delete()
             session.commit()
 
     def has_path_entry(self, path: Path) -> bool:

tagstudio/src/core/utils/missing_files.py

@@ -60,10 +60,7 @@ def fix_missing_files(self) -> Iterator[int]:
                 self.missing_files.remove(entry)
             yield i
 
-    def execute_deletion(self) -> Iterator[int]:
-        for i, missing in enumerate(self.missing_files, start=1):
-            # TODO - optimize this by removing multiple entries at once
-            self.library.remove_entries([missing.id])
-            yield i
+    def execute_deletion(self) -> None:
+        self.library.remove_entries(list(map(lambda missing: missing.id, self.missing_files)))
 
         self.missing_files = []

tagstudio/src/qt/modals/delete_unlinked.py

@@ -4,7 +4,7 @@
 
 import typing
 
-from PySide6.QtCore import Qt, Signal
+from PySide6.QtCore import Qt, QThreadPool, Signal
 from PySide6.QtGui import QStandardItem, QStandardItemModel
 from PySide6.QtWidgets import (
     QHBoxLayout,
@@ -15,6 +15,7 @@
     QWidget,
 )
 from src.core.utils.missing_files import MissingRegistry
+from src.qt.helpers.custom_runnable import CustomRunnable
 from src.qt.translations import Translations
 from src.qt.widgets.progress import ProgressWidget
 
@@ -95,8 +96,18 @@ def displayed_text(x):
         pw = ProgressWidget(
             cancel_button_text=None,
             minimum=0,
-            maximum=self.tracker.missing_files_count,
+            maximum=0,
         )
         Translations.translate_with_setter(pw.setWindowTitle, "entries.unlinked.delete.deleting")
-
-        pw.from_iterable_function(self.tracker.execute_deletion, displayed_text, self.done.emit)
+        Translations.translate_with_setter(pw.update_label, "entries.unlinked.delete.deleting")
+        pw.show()
+
+        r = CustomRunnable(self.tracker.execute_deletion)
+        QThreadPool.globalInstance().start(r)
+        r.done.connect(
+            lambda: (
+                pw.hide(),
+                pw.deleteLater(),
+                self.done.emit(),
+            )
+        )