fix(database): handle deletion of multiple records in chunks

* Implemented chunking for delete operations to avoid SQL variable limits.
* Ensured that the deletion process is efficient and does not exceed SQLite's constraints.
* Updated logging to reflect the number of records deleted.

Author: atasoglu

benchmarks/runner.py

@@ -29,50 +29,51 @@ def run_benchmark_suite(
         )
         client = SQLiteVecClient(table="benchmark", db_path=db_path)
 
-        # Create table
-        dim = config["dimension"]
-        distance = config["distance"]
-        client.create_table(dim=dim, distance=distance)
-
-        # Generate data
-        texts = generate_texts(dataset_size)
-        embeddings = generate_embeddings(dataset_size, dim)
-        metadata = generate_metadata(dataset_size)
-
-        # Benchmark: Add
-        print(f"  Benchmarking add ({dataset_size} records)...")
-        results.append(benchmark_add(client, texts, embeddings, metadata))
-
-        # Get rowids for subsequent operations
-        rowids = list(range(1, dataset_size + 1))
-
-        # Benchmark: Get Many
-        print(f"  Benchmarking get_many ({dataset_size} records)...")
-        results.append(benchmark_get_many(client, rowids))
-
-        # Benchmark: Similarity Search
-        print("  Benchmarking similarity_search...")
-        query_emb = [0.5] * dim
-        iterations = config["similarity_search"]["iterations"]
-        for top_k in config["similarity_search"]["top_k_values"]:
-            results.append(
-                benchmark_similarity_search(client, query_emb, top_k, iterations)
-            )
-
-        # Benchmark: Update Many
-        print(f"  Benchmarking update_many ({dataset_size} records)...")
-        new_texts = [f"updated_{i}" for i in range(dataset_size)]
-        results.append(benchmark_update_many(client, rowids, new_texts))
-
-        # Benchmark: Get All
-        print(f"  Benchmarking get_all ({dataset_size} records)...")
-        batch_size = config["batch_size"]
-        results.append(benchmark_get_all(client, dataset_size, batch_size))
-
-        # Benchmark: Delete Many
-        print(f"  Benchmarking delete_many ({dataset_size} records)...")
-        results.append(benchmark_delete_many(client, rowids))
-
-        client.close()
+        try:
+            # Create table
+            dim = config["dimension"]
+            distance = config["distance"]
+            client.create_table(dim=dim, distance=distance)
+
+            # Generate data
+            texts = generate_texts(dataset_size)
+            embeddings = generate_embeddings(dataset_size, dim)
+            metadata = generate_metadata(dataset_size)
+
+            # Benchmark: Add
+            print(f"  Benchmarking add ({dataset_size} records)...")
+            results.append(benchmark_add(client, texts, embeddings, metadata))
+
+            # Get rowids for subsequent operations
+            rowids = list(range(1, dataset_size + 1))
+
+            # Benchmark: Get Many
+            print(f"  Benchmarking get_many ({dataset_size} records)...")
+            results.append(benchmark_get_many(client, rowids))
+
+            # Benchmark: Similarity Search
+            print("  Benchmarking similarity_search...")
+            query_emb = [0.5] * dim
+            iterations = config["similarity_search"]["iterations"]
+            for top_k in config["similarity_search"]["top_k_values"]:
+                results.append(
+                    benchmark_similarity_search(client, query_emb, top_k, iterations)
+                )
+
+            # Benchmark: Update Many
+            print(f"  Benchmarking update_many ({dataset_size} records)...")
+            new_texts = [f"updated_{i}" for i in range(dataset_size)]
+            results.append(benchmark_update_many(client, rowids, new_texts))
+
+            # Benchmark: Get All
+            print(f"  Benchmarking get_all ({dataset_size} records)...")
+            batch_size = config["batch_size"]
+            results.append(benchmark_get_all(client, dataset_size, batch_size))
+
+            # Benchmark: Delete Many
+            print(f"  Benchmarking delete_many ({dataset_size} records)...")
+            results.append(benchmark_delete_many(client, rowids))
+        finally:
+            client.close()
 
     return results

sqlite_vec_client/base.py

@@ -456,15 +456,25 @@ def delete_many(self, rowids: list[int]) -> int:
         if not rowids:
             return 0
         logger.debug(f"Deleting {len(rowids)} records")
-        placeholders = ",".join(["?"] * len(rowids))
+
+        # SQLite has a limit on SQL variables (typically 999 or 32766)
+        # Split into chunks to avoid "too many SQL variables" error
+        chunk_size = 500
         cur = self.connection.cursor()
-        cur.execute(
-            f"DELETE FROM {self.table} WHERE rowid IN ({placeholders})",
-            rowids,
-        )
+        deleted_count = 0
+
+        for i in range(0, len(rowids), chunk_size):
+            chunk = rowids[i : i + chunk_size]
+            placeholders = ",".join(["?"] * len(chunk))
+            cur.execute(
+                f"DELETE FROM {self.table} WHERE rowid IN ({placeholders})",
+                chunk,
+            )
+            deleted_count += cur.rowcount
+
         if not self._in_transaction:
             self.connection.commit()
-        deleted_count = cur.rowcount
+
         logger.info(f"Deleted {deleted_count} records from table '{self.table}'")
         return deleted_count