feat(sentences): extract sentence content from sql.

Avoid to fetch the entire chunk to extract the content

Author: Daniele Briggi

src/sqlite_rag/database.py

@@ -76,19 +76,19 @@ def _create_schema(conn: sqlite3.Connection, settings: Settings):
         )
 
         # TODO: this table is not ready for sqlite-sync, it uses the id AUTOINCREMENT
-        cursor.execute(
+        cursor.executescript(
             """
             CREATE TABLE IF NOT EXISTS chunks (
                 id INTEGER PRIMARY KEY AUTOINCREMENT,
                 document_id TEXT,
                 content TEXT,
-                embedding BLOB,
-                FOREIGN KEY (document_id) REFERENCES documents (id) ON DELETE CASCADE
+                embedding BLOB
             );
+            CREATE INDEX IF NOT EXISTS idx_chunks_document_id ON chunks (document_id);
         """
         )
 
-        cursor.execute(
+        cursor.executescript(
             """
             CREATE TABLE IF NOT EXISTS sentences (
                 id TEXT PRIMARY KEY,
@@ -97,7 +97,8 @@ def _create_schema(conn: sqlite3.Connection, settings: Settings):
                 embedding BLOB,
                 start_offset INTEGER,
                 end_offset INTEGER
-            )
+            );
+            CREATE INDEX IF NOT EXISTS idx_sentences_chunk_id ON sentences (chunk_id);
             """
         )
 

src/sqlite_rag/engine.py

@@ -270,22 +270,27 @@ def search_sentences(
                 SELECT
                     v.rowid AS sentence_id,
                     row_number() OVER (ORDER BY v.distance) AS rank_number,
-                    v.distance,
-                    sentences.start_offset as sentence_start_offset,
-                    sentences.end_offset as sentence_end_offset
+                    v.distance
                 FROM {vector_scan_type}('sentences', 'embedding', :query_embedding) AS v
                     JOIN sentences ON sentences.rowid = v.rowid
                 WHERE sentences.chunk_id = :chunk_id
-                ORDER BY rank_number ASC
                 LIMIT :top_k
             )
             SELECT
                 sentence_id,
-                sentence_start_offset,
-                sentence_end_offset,
+                -- Extract sentence directly from document content
+                COALESCE(
+                    substr(chunks.content, sentences.start_offset + 1, sentences.end_offset - sentences.start_offset),
+                    ""
+                ) AS content,
+                sentences.start_offset AS sentence_start_offset,
+                sentences.end_offset AS sentence_end_offset,
                 rank_number,
                 distance
             FROM vec_matches
+                JOIN sentences ON sentences.rowid = vec_matches.sentence_id
+                JOIN chunks ON chunks.id = sentences.chunk_id
+            ORDER BY rank_number ASC
             """,  # nosec B608
             {
                 "query_embedding": query_embedding,
@@ -301,6 +306,7 @@ def search_sentences(
                 SentenceResult(
                     id=row["sentence_id"],
                     chunk_id=chunk_id,
+                    content=row["content"].strip(),
                     rank=row["rank_number"],
                     distance=row["distance"],
                     start_offset=row["sentence_start_offset"],

src/sqlite_rag/formatters.py

@@ -2,12 +2,19 @@
 """Output formatters for CLI search results."""
 
 from abc import ABC, abstractmethod
-from typing import List, Optional
+from typing import List
 
 import typer
 
 from .models.document_result import DocumentResult
 
+# Display constants
+BOX_CONTENT_WIDTH = 75
+BOX_TOTAL_WIDTH = 77
+SNIPPET_MAX_LENGTH = 400
+SENTENCE_PREVIEW_LENGTH = 50
+MAX_SENTENCES_DISPLAY = 5
+
 
 class SearchResultFormatter(ABC):
     """Base class for search result formatters."""
@@ -40,7 +47,10 @@ def _get_file_icon(self, uri: str) -> str:
         return "📄"
 
     def _clean_and_wrap_snippet(
-        self, snippet: str, width: int = 75, max_length: int = 400
+        self,
+        snippet: str,
+        width: int = BOX_CONTENT_WIDTH,
+        max_length: int = SNIPPET_MAX_LENGTH,
     ) -> List[str]:
         """Clean snippet and wrap to specified width with max length limit."""
         # Clean the snippet
@@ -69,7 +79,9 @@ def _clean_and_wrap_snippet(
 
         return lines
 
-    def _format_uri_display(self, uri: str, icon: str, max_width: int = 75) -> str:
+    def _format_uri_display(
+        self, uri: str, icon: str, max_width: int = BOX_CONTENT_WIDTH
+    ) -> str:
         """Format URI for display with icon and truncation."""
         if not uri:
             return ""
@@ -82,7 +94,15 @@ def _format_uri_display(self, uri: str, icon: str, max_width: int = 75) -> str:
 
 
 class BoxedFormatter(SearchResultFormatter):
-    """Base class for boxed result formatters."""
+    """Boxed formatter for search results with optional debug information."""
+
+    def __init__(self, show_debug: bool = False):
+        """Initialize formatter.
+
+        Args:
+            show_debug: Whether to show debug information and sentence details
+        """
+        self.show_debug = show_debug
 
     def format_results(self, results: List[DocumentResult], query: str) -> None:
         if not results:
@@ -98,56 +118,39 @@ def format_results(self, results: List[DocumentResult], query: str) -> None:
     def _format_single_result(self, doc: DocumentResult, idx: int) -> None:
         """Format a single result with box layout."""
         icon = self._get_file_icon(doc.document.uri or "")
+        snippet_text = doc.get_preview(max_chars=SNIPPET_MAX_LENGTH)
+        snippet_lines = self._clean_and_wrap_snippet(snippet_text)
 
-        # Get snippet from DocumentResult (handles sentence-based preview automatically)
-        snippet_text = doc.get_preview(max_chars=400)
-
-        snippet_lines = self._clean_and_wrap_snippet(
-            snippet_text, width=75, max_length=400
-        )
-
-        # Draw the result box header
-        header = f"┌─ Result #{idx} " + "─" * (67 - len(str(idx)))
+        # Draw box header
+        header = f"┌─ Result #{idx} " + "─" * (BOX_TOTAL_WIDTH - 10 - len(str(idx)))
         typer.echo(header)
 
-        # Display URI if available
+        # Display URI and debug info
         if doc.document.uri:
-            uri_display = self._format_uri_display(doc.document.uri, icon, 75)
-            typer.echo(f"│ {uri_display:<75}│")
+            uri_display = self._format_uri_display(doc.document.uri, icon)
+            typer.echo(f"│ {uri_display:<{BOX_CONTENT_WIDTH}}│")
 
-            # Add debug info if needed
-            debug_line = self._get_debug_line(doc)
-            if debug_line:
-                typer.echo(debug_line)
+            if self.show_debug:
+                self._print_debug_line(doc)
 
-            typer.echo("├" + "─" * 77 + "┤")
-        elif self._should_show_debug():
-            debug_line = self._get_debug_line(doc)
-            if debug_line:
-                typer.echo(debug_line)
-                typer.echo("├" + "─" * 77 + "┤")
+            typer.echo("├" + "─" * BOX_TOTAL_WIDTH + "┤")
+        elif self.show_debug:
+            self._print_debug_line(doc)
+            typer.echo("├" + "─" * BOX_TOTAL_WIDTH + "┤")
 
         # Display snippet
         for line in snippet_lines:
-            typer.echo(f"│ {line:<75} │")
+            typer.echo(f"│ {line:<{BOX_CONTENT_WIDTH}} │")
 
-        typer.echo("└" + "─" * 77 + "┘")
-        typer.echo()
+        # Display sentence details in debug mode
+        if self.show_debug and doc.sentences:
+            self._print_sentence_details(doc)
 
-    def _get_debug_line(self, doc: DocumentResult) -> Optional[str]:
-        """Get debug information line. Override in subclasses."""
-        return None
-
-    def _should_show_debug(self) -> bool:
-        """Whether to show debug information. Override in subclasses."""
-        return False
-
-
-class BoxedDebugFormatter(BoxedFormatter):
-    """Modern detailed formatter with debug information in boxes."""
+        typer.echo("└" + "─" * BOX_TOTAL_WIDTH + "┘")
+        typer.echo()
 
-    def _get_debug_line(self, doc: DocumentResult) -> str:
-        """Format debug metrics line."""
+    def _print_debug_line(self, doc: DocumentResult) -> None:
+        """Print debug metrics line."""
         combined = (
             f"{doc.combined_rank:.5f}" if doc.combined_rank is not None else "N/A"
         )
@@ -161,88 +164,36 @@ def _get_debug_line(self, doc: DocumentResult) -> str:
             if doc.fts_rank is not None
             else "N/A"
         )
-        return f"│ Combined: {combined} │ Vector: {vec_info} │ FTS: {fts_info}"
-
-    def _should_show_debug(self) -> bool:
-        return True
-
-    def _format_single_result(self, doc: DocumentResult, idx: int) -> None:
-        """Format a single result with box layout including sentence summary."""
-        icon = self._get_file_icon(doc.document.uri or "")
-
-        # Get snippet from DocumentResult (handles sentence-based preview automatically)
-        snippet_text = doc.get_preview(max_chars=400)
-
-        snippet_lines = self._clean_and_wrap_snippet(
-            snippet_text, width=75, max_length=400
-        )
+        debug_line = f"│ Combined: {combined} │ Vector: {vec_info} │ FTS: {fts_info}"
+        typer.echo(debug_line)
 
-        # Draw the result box header
-        header = f"┌─ Result #{idx} " + "─" * (67 - len(str(idx)))
-        typer.echo(header)
+    def _print_sentence_details(self, doc: DocumentResult) -> None:
+        """Print sentence-level details."""
+        typer.echo("├" + "─" * BOX_TOTAL_WIDTH + "┤")
+        typer.echo(f"│ Sentences:{' ' * (BOX_CONTENT_WIDTH - 10)}│")
 
-        # Display URI if available
-        if doc.document.uri:
-            uri_display = self._format_uri_display(doc.document.uri, icon, 75)
-            typer.echo(f"│ {uri_display:<75}│")
-
-            # Add debug info
-            debug_line = self._get_debug_line(doc)
-            if debug_line:
-                typer.echo(debug_line)
-
-            typer.echo("├" + "─" * 77 + "┤")
-        elif self._should_show_debug():
-            debug_line = self._get_debug_line(doc)
-            if debug_line:
-                typer.echo(debug_line)
-                typer.echo("├" + "─" * 77 + "┤")
-
-        # Display snippet preview
-        for line in snippet_lines:
-            typer.echo(f"│ {line:<75} │")
-
-        # Display sentence details if available
-        if doc.sentences:
-            typer.echo("├" + "─" * 77 + "┤")
-            typer.echo(
-                "│ Sentences:                                                                │"
+        for sentence in doc.sentences[:MAX_SENTENCES_DISPLAY]:
+            distance_str = (
+                f"{sentence.distance:.6f}" if sentence.distance is not None else "N/A"
             )
-
-            for sentence in doc.sentences[:5]:  # Show max 5 sentences
-                distance_str = (
-                    f"{sentence.distance:.6f}"
-                    if sentence.distance is not None
-                    else "N/A"
-                )
-                rank_str = f"#{sentence.rank}" if sentence.rank is not None else "N/A"
-
-                # Extract sentence preview (first 50 chars)
-                if (
-                    sentence.start_offset is not None
-                    and sentence.end_offset is not None
-                ):
-                    sentence_text = doc.chunk_content[
-                        sentence.start_offset : sentence.end_offset
-                    ].strip()
-                    # Truncate and clean for display
-                    sentence_preview = sentence_text.replace("\n", " ").replace(
-                        "\r", ""
+            rank_str = f"#{sentence.rank}" if sentence.rank is not None else "N/A"
+
+            # Extract sentence preview
+            if sentence.start_offset is not None and sentence.end_offset is not None:
+                sentence_text = doc.chunk_content[
+                    sentence.start_offset : sentence.end_offset
+                ].strip()
+                sentence_preview = sentence_text.replace("\n", " ").replace("\r", "")
+                if len(sentence_preview) > SENTENCE_PREVIEW_LENGTH:
+                    sentence_preview = (
+                        sentence_preview[: SENTENCE_PREVIEW_LENGTH - 3] + "..."
                     )
-                    if len(sentence_preview) > 50:
-                        sentence_preview = sentence_preview[:47] + "..."
-                else:
-                    sentence_preview = "[No offset info]"
-
-                # Format sentence line
-                sentence_line = (
-                    f"│   {rank_str:>3} ({distance_str}) | {sentence_preview}"
-                )
-                # Pad to 78 chars and add closing border
-                typer.echo(sentence_line.ljust(78) + " │")
-
-        typer.echo("└" + "─" * 77 + "┘")
-        typer.echo()
+            else:
+                sentence_preview = "[No offset info]"
+
+            # Format and print sentence line
+            sentence_line = f"│   {rank_str:>3} ({distance_str}) | {sentence_preview}"
+            typer.echo(sentence_line.ljust(BOX_TOTAL_WIDTH + 1) + " │")
 
 
 class TableDebugFormatter(SearchResultFormatter):
@@ -312,10 +263,15 @@ def _print_table_row(self, idx: int, doc: DocumentResult) -> None:
 def get_formatter(
     debug: bool = False, table_view: bool = False
 ) -> SearchResultFormatter:
-    """Factory function to get the appropriate formatter."""
+    """Factory function to get the appropriate formatter.
+
+    Args:
+        debug: Show debug information and sentence details
+        table_view: Use table format instead of boxed format
+
+    Returns:
+        SearchResultFormatter instance
+    """
     if table_view:
         return TableDebugFormatter()
-    elif debug:
-        return BoxedDebugFormatter()
-    else:
-        return BoxedFormatter()
+    return BoxedFormatter(show_debug=debug)