Fix SQLite concurrency issues with WAL mode and timeouts

backend/app/database/connection.py

@@ -7,26 +7,37 @@
 @contextmanager
 def get_db_connection() -> Generator[sqlite3.Connection, None, None]:
     """
-    SQLite connection context manager with all integrity constraints enforced.
+    SQLite connection context manager with concurrency optimization and integrity constraints.
 
-    - Enables all major relational integrity PRAGMAs
+    - Enables WAL mode for better concurrent access
+    - Configures timeouts and concurrency PRAGMAs
+    - Enforces all major relational integrity PRAGMAs
     - Works for both single and multi-step transactions
     - Automatically commits on success or rolls back on failure
     """
-    conn = sqlite3.connect(DATABASE_PATH)
-
-    # --- Strict enforcement of all relational and logical rules ---
-    conn.execute("PRAGMA foreign_keys = ON;")  # Enforce FK constraints
-    conn.execute("PRAGMA ignore_check_constraints = OFF;")  # Enforce CHECK constraints
-    conn.execute("PRAGMA recursive_triggers = ON;")  # Allow nested triggers
-    conn.execute("PRAGMA defer_foreign_keys = OFF;")  # Immediate FK checking
-    conn.execute("PRAGMA case_sensitive_like = ON;")  # Make LIKE case-sensitive
-
+    conn = None
     try:
+        conn = sqlite3.connect(DATABASE_PATH, timeout=30.0)
+
+        # --- Concurrency and performance optimizations ---
+        conn.execute("PRAGMA journal_mode = WAL;")  # Enable WAL mode for better concurrency
+        conn.execute("PRAGMA synchronous = NORMAL;")  # Balance safety and performance
+        conn.execute("PRAGMA cache_size = -64000;")  # 64MB cache
+        conn.execute("PRAGMA temp_store = MEMORY;")  # Store temp tables in memory
+        conn.execute("PRAGMA mmap_size = 268435456;")  # 256MB memory-mapped I/O
+        conn.execute("PRAGMA busy_timeout = 30000;")  # 30 second timeout for locks
+
+        # --- Strict enforcement of all relational and logical rules ---
+        conn.execute("PRAGMA foreign_keys = ON;")  # Enforce FK constraints
+        conn.execute("PRAGMA ignore_check_constraints = OFF;")  # Enforce CHECK constraints
+        conn.execute("PRAGMA defer_foreign_keys = OFF;")  # Immediate FK checking
+
         yield conn
         conn.commit()
     except Exception:
-        conn.rollback()
+        if conn:
+            conn.rollback()
         raise
     finally:
-        conn.close()
+        if conn:
+            conn.close()

backend/app/database/images.py

@@ -6,6 +6,7 @@
 from app.config.settings import (
     DATABASE_PATH,
 )
+from app.database.connection import get_db_connection
 from app.logging.setup_logging import get_logger
 
 # Initialize logger
@@ -43,48 +44,56 @@ class UntaggedImageRecord(TypedDict):
 
 
 def _connect() -> sqlite3.Connection:
-    conn = sqlite3.connect(DATABASE_PATH)
-    # Ensure ON DELETE CASCADE and other FKs are enforced
+    """Legacy connection function - use get_db_connection() context manager instead"""
+    conn = sqlite3.connect(DATABASE_PATH, timeout=30.0)
+    # Basic concurrency settings for legacy usage
+    conn.execute("PRAGMA journal_mode = WAL;")
+    conn.execute("PRAGMA busy_timeout = 30000;")
     conn.execute("PRAGMA foreign_keys = ON")
     return conn
 
-
 def db_create_images_table() -> None:
+    """Create images and image_classes tables if they don't exist."""
     conn = _connect()
     cursor = conn.cursor()
 
-    # Create new images table with merged fields
-    cursor.execute(
+    try:
+        # Create new images table with merged fields
+        cursor.execute(
+            """
+            CREATE TABLE IF NOT EXISTS images (
+                id TEXT PRIMARY KEY,
+                path VARCHAR UNIQUE,
+                folder_id INTEGER,
+                thumbnailPath TEXT UNIQUE,
+                metadata TEXT,
+                isTagged BOOLEAN DEFAULT 0,
+                isFavourite BOOLEAN DEFAULT 0,
+                FOREIGN KEY (folder_id) REFERENCES folders(folder_id) ON DELETE CASCADE
+            )
         """
-        CREATE TABLE IF NOT EXISTS images (
-            id TEXT PRIMARY KEY,
-            path VARCHAR UNIQUE,
-            folder_id INTEGER,
-            thumbnailPath TEXT UNIQUE,
-            metadata TEXT,
-            isTagged BOOLEAN DEFAULT 0,
-            isFavourite BOOLEAN DEFAULT 0,
-            FOREIGN KEY (folder_id) REFERENCES folders(folder_id) ON DELETE CASCADE
         )
-    """
-    )
 
-    # Create new image_classes junction table
-    cursor.execute(
+        # Create new image_classes junction table
+        cursor.execute(
+            """
+            CREATE TABLE IF NOT EXISTS image_classes (
+                image_id TEXT,
+                class_id INTEGER,
+                PRIMARY KEY (image_id, class_id),
+                FOREIGN KEY (image_id) REFERENCES images(id) ON DELETE CASCADE,
+                FOREIGN KEY (class_id) REFERENCES mappings(class_id) ON DELETE CASCADE
+            )
         """
-        CREATE TABLE IF NOT EXISTS image_classes (
-            image_id TEXT,
-            class_id INTEGER,
-            PRIMARY KEY (image_id, class_id),
-            FOREIGN KEY (image_id) REFERENCES images(id) ON DELETE CASCADE,
-            FOREIGN KEY (class_id) REFERENCES mappings(class_id) ON DELETE CASCADE
         )
-    """
-    )
-
-    conn.commit()
-    conn.close()
 
+        conn.commit()
+    except Exception as e:
+        logger.error(f"Error creating images table: {e}")
+        conn.rollback()
+        raise
+    finally:
+        conn.close()
 
 def db_bulk_insert_images(image_records: List[ImageRecord]) -> bool:
     """Insert multiple image records in a single transaction."""
@@ -395,27 +404,23 @@ def db_delete_images_by_ids(image_ids: List[ImageId]) -> bool:
     finally:
         conn.close()
 
-
 def db_toggle_image_favourite_status(image_id: str) -> bool:
-    conn = sqlite3.connect(DATABASE_PATH)
-    cursor = conn.cursor()
-    try:
-        cursor.execute("SELECT id FROM images WHERE id = ?", (image_id,))
-        if not cursor.fetchone():
-            return False
-        cursor.execute(
-            """
-            UPDATE images
-            SET isFavourite = CASE WHEN isFavourite = 1 THEN 0 ELSE 1 END
-            WHERE id = ?
-            """,
-            (image_id,),
-        )
-        conn.commit()
-        return cursor.rowcount > 0
-    except Exception as e:
-        logger.error(f"Database error: {e}")
-        conn.rollback()
-        return False
-    finally:
-        conn.close()
+    with get_db_connection() as conn:
+        cursor = conn.cursor()
+        try:
+            cursor.execute("SELECT id FROM images WHERE id = ?", (image_id,))
+            if not cursor.fetchone():
+                return False
+            cursor.execute(
+                """
+                UPDATE images
+                SET isFavourite = CASE WHEN isFavourite = 1 THEN 0 ELSE 1 END
+                WHERE id = ?
+                """,
+                (image_id,),
+            )
+            return cursor.rowcount > 0
+        except Exception as e:
+            logger.error(f"Database error: {e}")
+            raise
+