fix: run checkpoint after DuckDB inserts (#145)

Author: lsorber

src/raglite/_eval.py

@@ -5,6 +5,7 @@
 
 import numpy as np
 from pydantic import BaseModel, ConfigDict, Field, field_validator
+from sqlalchemy import text
 from sqlmodel import Session, func, select
 from tqdm.auto import tqdm, trange
 
@@ -18,7 +19,7 @@
 from raglite._search import hybrid_search, retrieve_chunk_spans, vector_search
 
 
-def insert_evals(  # noqa: C901
+def insert_evals(  # noqa: C901, PLR0912
     *, num_evals: int = 100, max_contexts_per_eval: int = 20, config: RAGLiteConfig | None = None
 ) -> None:
     """Generate and insert evals into the database."""
@@ -172,6 +173,8 @@ class AnswerResponse(BaseModel):
             )
             session.add(eval_)
             session.commit()
+            if engine.dialect.name == "duckdb":
+                session.execute(text("CHECKPOINT;"))
 
 
 def answer_evals(

src/raglite/_insert.py

@@ -157,11 +157,14 @@ def insert_document(
             session.commit()
             if engine.dialect.name == "duckdb":
                 # DuckDB does not automatically update its keyword search index [1], so we do it
-                # manually after insertion.
+                # manually after insertion. Additionally, we synchronize data in the write-ahead log
+                # (WAL) to the database data file with the CHECKPOINT statement [2].
                 # [1] https://duckdb.org/docs/stable/extensions/full_text_search
+                # [2] https://duckdb.org/docs/stable/sql/statements/checkpoint.html
                 session.execute(
                     text("PRAGMA create_fts_index('chunk', 'id', 'body', overwrite = 1);")
                 )
                 session.commit()
+                session.execute(text("CHECKPOINT;"))
         pbar.update(1)
         pbar.close()

src/raglite/_query_adapter.py

@@ -1,6 +1,7 @@
 """Compute and update an optimal query adapter."""
 
 import numpy as np
+from sqlalchemy import text
 from sqlalchemy.orm.attributes import flag_modified
 from sqlmodel import Session, col, select
 from tqdm.auto import tqdm
@@ -162,3 +163,5 @@ def update_query_adapter(  # noqa: PLR0915, C901
         flag_modified(index_metadata, "metadata_")
         session.add(index_metadata)
         session.commit()
+        if engine.dialect.name == "duckdb":
+            session.execute(text("CHECKPOINT;"))