perf(pipeline): ottimizza embedding pipeline con chunking semplificato, batch sizes aumentati e cache LRU per query

Rimosso Chonkie per chunking single-pass con Semchunk, aggiunta cache tiktoken globale, eliminato token count ridondante, batch embedding 32 e qdrant 100, embedding model configurabile via EMBEDDING_MODEL env, cache LRU per query embeddings

Author: strawberry-code

api/routes/search.py

@@ -2,6 +2,7 @@
 
 import os
 import warnings
+from functools import lru_cache
 from typing import Optional
 
 from fastapi import APIRouter, HTTPException
@@ -17,6 +18,7 @@
 QDRANT_URL = os.getenv('QDRANT_URL', 'http://localhost:6333')
 QDRANT_API_KEY = os.getenv('QDRANT_API_KEY')
 OLLAMA_URL = os.getenv('OLLAMA_URL', 'http://localhost:11434')
+EMBEDDING_MODEL = os.getenv('EMBEDDING_MODEL', 'nomic-embed-text')
 
 
 def get_qdrant_client() -> QdrantClient:
@@ -26,22 +28,34 @@ def get_qdrant_client() -> QdrantClient:
     return QdrantClient(url=QDRANT_URL)
 
 
-def get_embedding(text: str, model: str = "nomic-embed-text", timeout: int = 30) -> list[float] | None:
-    """Generate embedding using Ollama."""
+@lru_cache(maxsize=1000)
+def _cached_embedding(text: str, model: str) -> tuple[float, ...] | None:
+    """Generate embedding with LRU cache (returns tuple for hashability)."""
     import requests
 
     try:
         response = requests.post(
             f"{OLLAMA_URL}/api/embeddings",
             json={"model": model, "prompt": text},
-            timeout=timeout
+            timeout=30
         )
         response.raise_for_status()
-        return response.json().get("embedding")
+        embedding = response.json().get("embedding")
+        return tuple(embedding) if embedding else None
     except Exception:
         return None
 
 
+def get_embedding(text: str, model: str = None, timeout: int = 30) -> list[float] | None:
+    """Generate embedding using Ollama with caching."""
+    if model is None:
+        model = EMBEDDING_MODEL
+
+    # Use cached version (returns tuple, convert back to list)
+    result = _cached_embedding(text, model)
+    return list(result) if result else None
+
+
 class SearchRequest(BaseModel):
     """Search request body."""
     query: str

config.yaml

@@ -9,24 +9,20 @@ extraction:
   max_file_size: 104857600  # 100MB
 
 chunking:
-  strategies:
-    markdown: semantic
-    code: syntax_aware
-    pdf: page_aware
-    default: semantic
+  # Single-pass semantic chunking with semchunk
   chunk_size: 512
   overlap: 50
   max_tokens: 2048
 
 embedding:
   provider: ollama
-  model: nomic-embed-text
-  batch_size: 10
+  model: nomic-embed-text  # Configurable via EMBEDDING_MODEL env
+  batch_size: 32  # Increased from 10 for better throughput
   # url: http://localhost:11434  # Defaults from env OLLAMA_URL
 
 qdrant:
-  collection: documentation  # Changed to match existing data
-  batch_size: 10
+  collection: documentation
+  batch_size: 100  # Increased from 10 for better throughput
   # url: http://localhost:6333  # Defaults from env QDRANT_URL
   # api_key: null  # Defaults from env QDRANT_API_KEY
 

lib/chunking.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python3
 """
-Semantic chunking utilities using Chonkie and semchunk.
-Implements two-level chunking strategy for optimal RAG performance.
+Semantic chunking utilities using semchunk.
+Simplified single-pass chunking for optimal RAG performance.
 """
 
 import logging
@@ -12,9 +12,9 @@
 logger = logging.getLogger(__name__)
 
 # Chunking configuration from environment variables
-# CHUNK_SIZE: Target chunk size in tokens (default: 400)
+# CHUNK_SIZE: Target chunk size in tokens (default: 512)
 # CHUNK_MAX_TOKENS: Maximum chunk size before re-chunking (default: 1500, safe for nomic-embed-text 2048 limit)
-CHUNK_SIZE = int(os.getenv('CHUNK_SIZE', '400'))
+CHUNK_SIZE = int(os.getenv('CHUNK_SIZE', '512'))
 CHUNK_MAX_TOKENS = int(os.getenv('CHUNK_MAX_TOKENS', '1500'))
 
 # Custom exception for chunking failures
@@ -23,22 +23,34 @@ class ChunkingError(RuntimeError):
     pass
 
 
+# Global tiktoken encoding cache for performance
+_TIKTOKEN_ENC = None
+
+
+def _get_tiktoken_encoding(encoding_name: str = "cl100k_base"):
+    """Get cached tiktoken encoding for performance."""
+    global _TIKTOKEN_ENC
+    if _TIKTOKEN_ENC is None:
+        _TIKTOKEN_ENC = tiktoken.get_encoding(encoding_name)
+    return _TIKTOKEN_ENC
+
+
 def count_tokens(text: str, encoding_name: str = "cl100k_base") -> int:
     """
-    Count tokens in text using tiktoken.
-    
+    Count tokens in text using tiktoken (cached).
+
     Args:
         text: Text to count tokens for
         encoding_name: Tiktoken encoding name
-        
+
     Returns:
         Number of tokens
     """
     try:
-        enc = tiktoken.get_encoding(encoding_name)
+        enc = _get_tiktoken_encoding(encoding_name)
         return len(enc.encode(text))
     except Exception as e:
-        logger.warning(f"Token counting failed: {e}, using word‑based fallback")
+        logger.warning(f"Token counting failed: {e}, using word-based fallback")
         # Fallback: approximate 1 token ≈ 1 word
         return len(text.split())
 
@@ -179,6 +191,63 @@ def fine_chunk_text(
         raise ChunkingError(str(e))
 
 
+def semchunk_text(
+    text: str,
+    target_tokens: int = 512,
+    overlap_tokens: int = 50
+) -> list[dict]:
+    """
+    Direct semantic chunking using semchunk.
+
+    Single-pass chunking that respects semantic boundaries.
+    Simpler and faster than two-level chunking.
+
+    Args:
+        text: Text to chunk
+        target_tokens: Target size for chunks (tokens)
+        overlap_tokens: Overlap between chunks (tokens)
+
+    Returns:
+        List of chunk dictionaries with metadata
+    """
+    if not text or len(text.strip()) == 0:
+        return []
+
+    try:
+        from semchunk import chunkerify
+
+        # Create chunker with cached tiktoken encoding
+        chunker = chunkerify("cl100k_base", chunk_size=target_tokens)
+
+        # Chunk the text directly
+        chunk_texts = chunker(text, overlap=overlap_tokens)
+
+        # Build chunk dictionaries with metadata
+        chunks = []
+        for idx, chunk_text in enumerate(chunk_texts):
+            if not chunk_text or len(chunk_text.strip()) == 0:
+                continue
+
+            token_count = count_tokens(chunk_text)
+            chunks.append({
+                'text': chunk_text,
+                'semantic_block_index': 0,  # Single block for direct chunking
+                'chunk_index': idx,
+                'token_count': token_count,
+                'chunking_method': 'semchunk'
+            })
+
+        logger.info(f"Semchunk created {len(chunks)} chunks from text")
+        return chunks
+
+    except ImportError as e:
+        logger.error("Semchunk not installed, cannot perform chunking")
+        raise ChunkingError("Semchunk not installed")
+    except Exception as e:
+        logger.warning(f"Semchunk failed: {e}")
+        raise ChunkingError(str(e))
+
+
 def _fallback_chunk(
     blocks: list[str],
     target_tokens: int,
@@ -244,16 +313,15 @@ def create_chunks(
     max_tokens: int = None
 ) -> list[dict]:
     """
-    Create chunks from text using two-level semantic chunking (chonkie + semchunk).
+    Create chunks from text using semantic chunking (semchunk only).
 
-    Pipeline:
-    1. Chonkie TokenChunker: creates macro-semantic blocks (2x target size)
-    2. Semchunk: refines into fine-grained embedding-ready chunks
-    3. Filter: removes too short/long chunks
+    Simplified pipeline:
+    1. Semchunk: creates embedding-ready chunks respecting semantic boundaries
+    2. Filter: removes too short/long chunks
 
     Args:
         text: Text to chunk
-        chunk_size: Target chunk size in tokens (default: CHUNK_SIZE env var or 400)
+        chunk_size: Target chunk size in tokens (default: CHUNK_SIZE env var or 512)
         chunk_overlap: Overlap between chunks in tokens (default: 50)
         min_tokens: Minimum chunk size to keep (default: 0)
         max_tokens: Maximum chunk size before re-chunking (default: CHUNK_MAX_TOKENS env var or 1500)
@@ -271,32 +339,26 @@ def create_chunks(
         return []
 
     try:
-        # Level 1: Chonkie semantic chunking (macro blocks)
-        macro_chunks = semantic_chunk_text(
+        # Direct semchunk - no need for two-level chunking
+        # Semchunk already handles semantic boundaries well
+        chunks = semchunk_text(
             text,
-            chunk_size=chunk_size * 2,  # Larger blocks first
-            chunk_overlap=chunk_overlap
-        )
-
-        if not macro_chunks:
-            logger.warning("No macro chunks created, using fallback")
-            return _fallback_chunk([text], chunk_size, chunk_overlap)
-
-        # Level 2: Semchunk fine-grained chunking
-        fine_chunks = fine_chunk_text(
-            macro_chunks,
             target_tokens=chunk_size,
             overlap_tokens=chunk_overlap
         )
 
-        # Level 3: Filter and validate
+        if not chunks:
+            logger.warning("No chunks created, using fallback")
+            return _fallback_chunk([text], chunk_size, chunk_overlap)
+
+        # Filter and validate
         valid_chunks = filter_chunks(
-            fine_chunks,
+            chunks,
             min_tokens=min_tokens,
             max_tokens=max_tokens
         )
 
-        logger.info(f"Created {len(valid_chunks)} chunks (chonkie+semchunk pipeline)")
+        logger.info(f"Created {len(valid_chunks)} chunks (semchunk pipeline)")
         return valid_chunks
 
     except ChunkingError as e:
@@ -319,7 +381,7 @@ def filter_chunks(
         chunks: List of chunk dictionaries
         min_tokens: Minimum token count - 0 = keep all (default)
         max_tokens: Maximum token count (default: CHUNK_MAX_TOKENS env var or 1500)
-        
+
     Returns:
         Filtered list of valid chunks
     """
@@ -329,7 +391,11 @@ def filter_chunks(
     valid_chunks = []
 
     for chunk in chunks:
-        token_count = chunk.get('token_count', count_tokens(chunk['text']))
+        # Use cached token_count if available, only count if missing
+        token_count = chunk.get('token_count')
+        if token_count is None:
+            token_count = count_tokens(chunk['text'])
+            chunk['token_count'] = token_count  # Cache for later use
 
         if token_count < min_tokens:
             logger.debug(f"Discarding too short chunk: {token_count} tokens")

lib/config.py

@@ -24,19 +24,9 @@ class ExtractionConfig(BaseModel):
     max_file_size: int = Field(default=100 * 1024 * 1024, description="Maximum file size in bytes")
 
 
-class ChunkingStrategyConfig(BaseModel):
-    """Chunking strategies per file type."""
-
-    markdown: str = Field(default="semantic", description="Strategy for markdown files")
-    code: str = Field(default="syntax_aware", description="Strategy for code files")
-    pdf: str = Field(default="page_aware", description="Strategy for PDF files")
-    default: str = Field(default="semantic", description="Default strategy")
-
-
 class ChunkingConfig(BaseModel):
-    """Configuration for text chunking."""
+    """Configuration for text chunking (single-pass semchunk)."""
 
-    strategies: ChunkingStrategyConfig = Field(default_factory=ChunkingStrategyConfig)
     chunk_size: int = Field(default=512, description="Target chunk size in tokens")
     overlap: int = Field(default=50, description="Overlap between chunks in tokens")
     max_tokens: int = Field(default=2048, description="Maximum tokens per chunk (nomic-embed-text limit)")
@@ -47,7 +37,7 @@ class EmbeddingConfig(BaseModel):
 
     provider: str = Field(default="ollama", description="Embedding provider")
     model: str = Field(default="nomic-embed-text", description="Embedding model name")
-    batch_size: int = Field(default=10, description="Batch size for embedding")
+    batch_size: int = Field(default=32, description="Batch size for embedding (optimized)")
     url: Optional[str] = Field(default=None, description="Ollama/API URL")
 
     @field_validator('url')
@@ -68,7 +58,7 @@ class QdrantConfig(BaseModel):
     """Configuration for Qdrant vector database."""
 
     collection: str = Field(default="documentation", description="Collection name")
-    batch_size: int = Field(default=10, description="Batch upload size")
+    batch_size: int = Field(default=100, description="Batch upload size (optimized)")
     url: str = Field(default_factory=_get_qdrant_url, description="Qdrant URL")
     api_key: Optional[str] = Field(default_factory=_get_qdrant_api_key, description="API key if required")
 
@@ -259,24 +249,20 @@ def merge_cli_args(config: RagifyConfig, args: dict) -> RagifyConfig:
   max_file_size: 104857600  # 100MB
 
 chunking:
-  strategies:
-    markdown: semantic
-    code: syntax_aware
-    pdf: page_aware
-    default: semantic
+  # Single-pass semantic chunking with semchunk
   chunk_size: 512
   overlap: 50
   max_tokens: 2048
 
 embedding:
   provider: ollama
-  model: nomic-embed-text
-  batch_size: 10
+  model: nomic-embed-text  # Configurable via EMBEDDING_MODEL env
+  batch_size: 32  # Optimized for throughput
   # url: http://localhost:11434  # Defaults from env OLLAMA_URL
 
 qdrant:
   collection: documentation
-  batch_size: 10
+  batch_size: 100  # Optimized for throughput
   # url: http://localhost:6333  # Defaults from env QDRANT_URL
   # api_key: null  # Defaults from env QDRANT_API_KEY