feat(context): new option

Author: Daniele Briggi

src/sqlite_rag/settings.py

@@ -18,7 +18,7 @@ class Settings:
     model_options: str = ""
     # See: https://github.com/sqliteai/sqlite-ai/blob/main/API.md#llm_context_createoptions-text
     model_context_options: str = (
-        "generate_embedding=1,normalize_embedding=1,pooling_type=mean"
+        "generate_embedding=1,normalize_embedding=1,pooling_type=mean,embedding_type=FLOAT16"
     )
 
     vector_type: str = "FLOAT16"

src/sqlite_rag/sqliterag.py

@@ -73,13 +73,16 @@ def add(
         self._engine.create_new_context()
 
         processed = 0
-        self._logger.info(f"Processing {len(files_to_process)} files...")
+        total_to_process = len(files_to_process)
+        self._logger.info(f"Processing {total_to_process} files...")
         try:
-            for file_path in files_to_process:
+            for i, file_path in enumerate(files_to_process):
                 content = FileReader.parse_file(file_path)
 
                 if not content:
-                    self._logger.warning(f"Skipping empty file: {file_path}")
+                    self._logger.warning(
+                        f"{i+1}/{total_to_process} Skipping empty file: {file_path}"
+                    )
                     continue
 
                 uri = (
@@ -91,10 +94,12 @@ def add(
 
                 exists = self._repository.document_exists_by_hash(document.hash())
                 if exists:
-                    self._logger.info(f"Unchanged: {file_path}")
+                    self._logger.info(
+                        f"{i+1}/{total_to_process} Unchanged: {file_path}"
+                    )
                     continue
 
-                self._logger.info(f"Processing: {file_path}")
+                self._logger.info(f"{i+1}/{total_to_process} Processing: {file_path}")
                 document = self._engine.process(document)
 
                 self._repository.add_document(document)

tests/test_engine.py

@@ -1,5 +1,3 @@
-import pytest
-
 from sqlite_rag.chunker import Chunker
 from sqlite_rag.engine import Engine
 from sqlite_rag.models.chunk import Chunk
@@ -22,13 +20,13 @@ def test_search_with_empty_database(self, engine):
 
         assert len(results) == 0
 
-    @pytest.mark.skip(reason="Waiting for sqlite-ai context to be fixed")
     def test_search_with_semantic_and_fts(self, db_conn):
         # Arrange
         conn, settings = db_conn
 
         engine = Engine(conn, settings, Chunker(conn, settings))
         engine.load_model()
+        engine.create_new_context()
 
         doc1 = Document(
             content="The quick brown fox jumps over the lazy dog.",
@@ -60,13 +58,13 @@ def test_search_with_semantic_and_fts(self, db_conn):
         assert len(results) > 0
         assert doc3_id == results[0].document.id
 
-    @pytest.mark.skip(reason="Waiting for sqlite-ai context to be fixed")
     def test_search_semantic_result(self, db_conn):
         # Arrange
         conn, settings = db_conn
 
         engine = Engine(conn, settings, Chunker(conn, settings))
         engine.load_model()
+        engine.create_new_context()
 
         doc1 = Document(
             content="The quick brown fox jumps over the lazy dog.",