Improve tracking for scaling beyond 50 jobs

Author: rbeucher

src/access_mopper/templates/cmor_python_script.j2

@@ -53,7 +53,9 @@ def main():
         print(f'Processing {variable} with {len(input_files)} files')
 
         # Initialize tracker
-        tracker = TaskTracker(Path(db_path))
+        db_name = f"cmor_tasks_{variable.replace('.', '_')}.db"
+        db_path = Path(os.environ['CMOR_TRACKER_DB']).parent / db_name
+        tracker = TaskTracker(db_path)
         tracker.add_task(variable, experiment_id)
 
         if tracker.is_done(variable, experiment_id):

src/access_mopper/tracking.py

@@ -1,4 +1,6 @@
+import random
 import sqlite3
+import time
 from pathlib import Path
 from typing import Optional
 
@@ -13,6 +15,9 @@ def __init__(self, db_path: Optional[Path] = None):
         self._init_db()
 
     def _init_db(self):
+        # Enable WAL mode for better concurrent access
+        self.conn.execute("PRAGMA journal_mode=WAL")
+        self.conn.execute("PRAGMA synchronous=NORMAL")
         with self.conn:
             self.conn.execute(
                 """
@@ -84,3 +89,16 @@ def is_done(self, variable: str, experiment: str) -> bool:
         )
         row = cur.fetchone()
         return row is not None and row[0] == "done"
+
+    def _execute_with_retry(self, query, params=(), max_retries=5):
+        for attempt in range(max_retries):
+            try:
+                with self.conn:
+                    return self.conn.execute(query, params)
+            except sqlite3.OperationalError as e:
+                if "database is locked" in str(e) and attempt < max_retries - 1:
+                    # Exponential backoff with jitter
+                    delay = (2**attempt) + random.uniform(0, 1)
+                    time.sleep(delay)
+                    continue
+                raise