feat: versions and settings commands

Author: Daniele Briggi

README.md

@@ -1 +1,7 @@
 # sqlite-rag
+
+## Installation
+
+```bash
+pip install .[dev]
+```

bandit.yaml

@@ -1,3 +1,3 @@
 # https://bndit.readthedocs.io/en/latest/config.html
-skips: ['B101']
+skips: ['B101', 'B608']
 exclude_dirs: ['tests']

extensions/ai.so

@@ -12,7 +12,9 @@ dependencies = [
     "attrs",
     "typer",
     "huggingface_hub",
-    "markitdown[all]"
+    "markitdown[all]",
+    "sqlite-ai",
+    "sqliteai-vector"
 ]
 
 # .. or [dependency-groups] ?

extensions/vector.so

@@ -9,11 +9,11 @@
 class Chunker:
     def __init__(self, conn: sqlite3.Connection, settings: Settings):
         self._conn = conn
-        self.settings = settings
+        self._settings = settings
 
     def chunk(self, text: str) -> list[Chunk]:
         """Chunk text using Recursive Character Text Splitter."""
-        if self._get_token_count(text) <= self.settings.chunk_size:
+        if self._get_token_count(text) <= self._settings.chunk_size:
             return [Chunk(content=text)]
 
         return self._recursive_split(text)
@@ -55,7 +55,7 @@ def _split_text_with_separators(
         """Split text using hierarchical separators."""
         chunks = []
 
-        if self.settings.chunk_size <= self.settings.chunk_overlap:
+        if self._settings.chunk_size <= self._settings.chunk_overlap:
             raise ValueError("Chunk size must be greater than chunk overlap.")
 
         if not separators:
@@ -70,7 +70,7 @@ def _split_text_with_separators(
 
         # Reserve space for overlap
         effective_chunk_size = max(
-            1, self.settings.chunk_size - self.settings.chunk_overlap
+            1, self._settings.chunk_size - self._settings.chunk_overlap
         )
 
         splits = text.split(separator)
@@ -108,7 +108,7 @@ def _split_by_characters(self, text: str) -> List[Chunk]:
 
         # Reserve space for overlap
         effective_chunk_size = max(
-            1, self.settings.chunk_size - self.settings.chunk_overlap
+            1, self._settings.chunk_size - self._settings.chunk_overlap
         )
 
         total_tokens = self._get_token_count(text)
@@ -145,7 +145,7 @@ def _split_by_characters(self, text: str) -> List[Chunk]:
 
     def _apply_overlap(self, chunks: List[Chunk]) -> List[Chunk]:
         """Apply overlap between consecutive chunks."""
-        if len(chunks) <= 1 or self.settings.chunk_overlap <= 0:
+        if len(chunks) <= 1 or self._settings.chunk_overlap <= 0:
             return chunks
 
         overlapped_chunks = [chunks[0]]  # First chunk has no overlap
@@ -156,7 +156,7 @@ def _apply_overlap(self, chunks: List[Chunk]) -> List[Chunk]:
 
             # Get overlap text from end of previous chunk
             overlap_text = self._get_overlap_text(
-                prev_content, self.settings.chunk_overlap
+                prev_content, self._settings.chunk_overlap
             )
 
             if overlap_text:

pyproject.toml

@@ -2,10 +2,13 @@
 import json
 import shlex
 import sys
+from dataclasses import replace
 from typing import Optional
 
 import typer
 
+from sqlite_rag.settings import Settings
+
 from .sqliterag import SQLiteRag
 
 
@@ -26,10 +29,89 @@ def __call__(self, *args, **kwds):
 cli = CLI(app)
 
 
-@app.command()
-def set(settings: Optional[str] = typer.Argument(None)):
-    """Set the model and database path"""
-    pass
+@app.command("settings")
+def show_settings():
+    """Show current settings"""
+    rag = SQLiteRag.create()
+    current_settings = rag.get_settings()
+
+    typer.echo("Current settings:")
+    for key, value in current_settings.items():
+        typer.echo(f"  {key}: {value}")
+
+
+# TODO: separate store settings from SQLiteRag.create()?
+@app.command("set")
+def set_settings(
+    model_path_or_name: Optional[str] = typer.Option(
+        None, help="Path to the embedding model file or Hugging Face model name"
+    ),
+    model_config: Optional[str] = typer.Option(
+        None, help="Model configuration parameters"
+    ),
+    embedding_dim: Optional[int] = typer.Option(
+        None, help="Dimension of the embedding vectors"
+    ),
+    vector_type: Optional[str] = typer.Option(
+        None, help="Vector storage type (FLOAT16, FLOAT32, etc.)"
+    ),
+    other_vector_config: Optional[str] = typer.Option(
+        None, help="Additional vector configuration"
+    ),
+    chunk_size: Optional[int] = typer.Option(
+        None, help="Size of text chunks for processing"
+    ),
+    chunk_overlap: Optional[int] = typer.Option(
+        None, help="Token overlap between consecutive chunks"
+    ),
+    quantize_scan: Optional[bool] = typer.Option(
+        None, help="Whether to quantize vector for faster search"
+    ),
+    quantize_preload: Optional[bool] = typer.Option(
+        None, help="Whether to preload quantized vectors in memory for faster search"
+    ),
+    weight_fts: Optional[float] = typer.Option(
+        None, help="Weight for full-text search results"
+    ),
+    weight_vec: Optional[float] = typer.Option(
+        None, help="Weight for vector search results"
+    ),
+):
+    """Change default settings for the RAG system.
+
+    Update model configuration, embedding parameters, chunking settings,
+    and search weights. Only specify the options you want to change.
+    Use 'sqlite-rag settings' to view current values.
+    """
+    # Build updates dict from all provided parameters
+    updates = {
+        "model_path_or_name": model_path_or_name,
+        "model_config": model_config,
+        "embedding_dim": embedding_dim,
+        "vector_type": vector_type,
+        "other_vector_config": other_vector_config,
+        "chunk_size": chunk_size,
+        "chunk_overlap": chunk_overlap,
+        "quantize_scan": quantize_scan,
+        "quantize_preload": quantize_preload,
+        "weight_fts": weight_fts,
+        "weight_vec": weight_vec,
+    }
+
+    # Filter out None values (unset options)
+    updates = {k: v for k, v in updates.items() if v is not None}
+
+    if not updates:
+        typer.echo("No settings provided to update.")
+        show_settings()
+        return
+
+    # Create new settings with updated fields
+    new_settings = replace(Settings(), **updates)
+    SQLiteRag.create(settings=new_settings)
+
+    show_settings()
+    typer.echo("Settings updated.")
 
 
 @app.command()
@@ -42,14 +124,12 @@ def add(
         False,
         "--absolute-paths",
         help="Store absolute paths instead of relative paths",
-        is_flag=True,
     ),
     metadata: Optional[str] = typer.Option(
         None,
         "--metadata",
         help="Optional metadata in JSON format to associate with the document",
         metavar="JSON",
-        show_default=False,
     ),
 ):
     """Add a file path to the database"""
@@ -71,8 +151,6 @@ def add_text(
         "--metadata",
         help="Optional metadata in JSON format to associate with the document",
         metavar="JSON",
-        show_default=False,
-        prompt="Metadata (JSON format, e.g. {'author': 'John Doe', 'date': '2023-10-01'}'",
     ),
 ):
     """Add a text to the database"""

src/sqlite_rag/chunker.py

@@ -1,29 +1,40 @@
+import importlib
+import importlib.resources
 import sqlite3
-from pathlib import Path
 
 from .settings import Settings
 
 
 class Database:
     """Database initialization and schema management for SQLiteRag."""
 
+    @staticmethod
+    def new_connection(db_path: str = "./sqliterag.sqlite") -> sqlite3.Connection:
+        """Create a new SQLite connection to the specified database path."""
+        conn = sqlite3.connect(db_path)
+        conn.row_factory = sqlite3.Row
+        return conn
+
     @staticmethod
     def initialize(conn: sqlite3.Connection, settings: Settings) -> sqlite3.Connection:
         """Initialize the database with extensions and schema"""
         conn.enable_load_extension(True)
         try:
             conn.load_extension(
-                str(Path(__file__).parent.parent.parent / "extensions" / "ai")
+                str(importlib.resources.files("sqlite-vector.binaries") / "ai")
             )
             conn.load_extension(
-                str(Path(__file__).parent.parent.parent / "extensions" / "vector")
+                str(importlib.resources.files("sqlite-vector.binaries") / "vector")
             )
         except sqlite3.OperationalError as e:
             raise RuntimeError(
                 "Failed to load extensions: "
                 + str(e)
                 + """\n
-                Download from:
+                Install via pip:
+                    pip install sqlite-ai sqliteai-vector
+
+                See more:
                     sqlite-ai: https://github.com/sqliteai/sqlite-ai/releases
                     sqlite-vector: https://github.com/sqliteai/sqlite-vector/releases
                 """