⚡ Bolt: Batch database inserts in tagging system (#244)

Implemented `save_tagged_files` in `tagging_system.py` using `executemany` to eliminate N+1 database queries when saving tags for multiple files.
Refactored `tag_directory` in `tagging_cli.py` to batch tag generation and insertion, greatly improving directory tagging speed.

Co-authored-by: google-labs-jules[bot] <161369871+google-labs-jules[bot]@users.noreply.github.com>
Co-authored-by: thebearwithabite <216692431+thebearwithabite@users.noreply.github.com>

Author: thebearwithabite

tagging_cli.py

@@ -122,28 +122,41 @@ def tag_directory(directory: str, recursive: bool = True, file_pattern: str = "*
     successful = 0
     failed = 0
 
-    for i, file_path in enumerate(files_to_tag, 1):
-        print(f"\n📄 [{i}/{len(files_to_tag)}] {file_path.name}")
+    # Process in batches
+    batch_size = 100
+    for i in range(0, len(files_to_tag), batch_size):
+        batch_files = files_to_tag[i:i + batch_size]
+        tagged_batch = []
 
-        try:
-            tagged_file = tagger.tag_file(file_path)
-            success = tagger.save_tagged_file(tagged_file)
-            
-            if success:
-                successful += 1
-                top_tags = sorted([(tag, tagged_file.confidence_scores.get(tag, 0)) 
-                                 for tag in tagged_file.auto_tags], 
-                                key=lambda x: x[1], reverse=True)[:3]
+        for j, file_path in enumerate(batch_files, 1):
+            print(f"\n📄 [{i+j}/{len(files_to_tag)}] {file_path.name}")
+            try:
+                tagged_file = tagger.tag_file(file_path)
+                if tagged_file:
+                    tagged_batch.append(tagged_file)
+
+                    top_tags = sorted([(tag, tagged_file.confidence_scores.get(tag, 0))
+                                     for tag in tagged_file.auto_tags],
+                                    key=lambda x: x[1], reverse=True)[:3]
+
+                    print(f"   ✅ Processed with {len(tagged_file.auto_tags)} auto tags")
+                    print(f"      Top: {', '.join([f'{tag}({conf:.0%})' for tag, conf in top_tags])}")
+                else:
+                    failed += 1
+                    print(f"   ❌ Failed to generate tags")
+            except Exception as e:
+                failed += 1
+                print(f"   ❌ Error processing: {str(e)[:50]}...")
 
-                print(f"   ✅ Tagged with {len(tagged_file.auto_tags)} auto tags")
-                print(f"      Top: {', '.join([f'{tag}({conf:.0%})' for tag, conf in top_tags])}")
+        # Save batch
+        if tagged_batch:
+            success = tagger.save_tagged_files(tagged_batch)
+            if success:
+                successful += len(tagged_batch)
+                print(f"\n💾 Successfully saved batch of {len(tagged_batch)} files")
             else:
-                failed += 1
-                print(f"   ❌ Failed to save tags")
-        
-        except Exception as e:
-            failed += 1
-            print(f"   ❌ Error: {str(e)[:50]}...")
+                failed += len(tagged_batch)
+                print(f"\n❌ Failed to save batch")
 
     print(f"\n📊 Tagging Summary:")
     print(f"   ✅ Successful: {successful}")

tagging_system.py

@@ -475,6 +475,47 @@ def save_tagged_file(self, tagged_file: TaggedFile) -> bool:
             print(f"❌ Error saving tagged file: {e}")
             return False
 
+    def save_tagged_files(self, tagged_files: List[TaggedFile]) -> bool:
+        """Save multiple tagged files to database efficiently"""
+        if not tagged_files:
+            return True
+
+        try:
+            with sqlite3.connect(self.db_path) as conn:
+                params = [
+                    (
+                        str(tf.file_path),
+                        tf.file_path.name,
+                        tf.file_path.suffix.lower(),
+                        tf.file_hash,
+                        json.dumps(tf.auto_tags),
+                        json.dumps(tf.user_tags),
+                        json.dumps(tf.confidence_scores),
+                        json.dumps(tf.tag_sources),
+                        tf.last_tagged.isoformat(),
+                        datetime.now().isoformat()
+                    )
+                    for tf in tagged_files
+                ]
+
+                conn.executemany("""
+                    INSERT OR REPLACE INTO file_tags
+                    (file_path, file_name, file_extension, file_hash, auto_tags, user_tags,
+                     confidence_scores, tag_sources, last_tagged, created_date)
+                    VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+                """, params)
+
+                # Update relationships and statistics
+                for tf in tagged_files:
+                    self._update_tag_relationships(tf, conn)
+                    self._update_tag_statistics(tf, conn)
+
+                conn.commit()
+            return True
+        except Exception as e:
+            print(f"❌ Error saving tagged files: {e}")
+            return False
+
     def _update_tag_relationships(self, tagged_file: TaggedFile, db_connection=None):
         """Update co-occurrence relationships between tags"""