Enhanced Ollama Embedding Configuration + List View improvements (#157)

* feature/configurable-ollama-embedder

* Add search functionality and card/list view toggle to processed projects

Author: sng-asyncfunc

api/config.py

@@ -108,6 +108,35 @@ def load_embedder_config():
 
     return embedder_config
 
+def get_embedder_config():
+    """
+    Get the current embedder configuration.
+
+    Returns:
+        dict: The embedder configuration with model_client resolved
+    """
+    return configs.get("embedder", {})
+
+def is_ollama_embedder():
+    """
+    Check if the current embedder configuration uses OllamaClient.
+
+    Returns:
+        bool: True if using OllamaClient, False otherwise
+    """
+    embedder_config = get_embedder_config()
+    if not embedder_config:
+        return False
+
+    # Check if model_client is OllamaClient
+    model_client = embedder_config.get("model_client")
+    if model_client:
+        return model_client.__name__ == "OllamaClient"
+
+    # Fallback: check client_class string
+    client_class = embedder_config.get("client_class", "")
+    return client_class == "OllamaClient"
+
 # Load repository and file filters configuration
 def load_repo_config():
     return load_json_config("repo.json")

api/data_pipeline.py

@@ -21,19 +21,25 @@
 # Maximum token limit for OpenAI embedding models
 MAX_EMBEDDING_TOKENS = 8192
 
-def count_tokens(text: str, local_ollama: bool = False) -> int:
+def count_tokens(text: str, is_ollama_embedder: bool = None) -> int:
     """
     Count the number of tokens in a text string using tiktoken.
 
     Args:
         text (str): The text to count tokens for.
-        local_ollama (bool, optional): Whether using local Ollama embeddings. Default is False.
+        is_ollama_embedder (bool, optional): Whether using Ollama embeddings.
+                                           If None, will be determined from configuration.
 
     Returns:
         int: The number of tokens in the text.
     """
     try:
-        if local_ollama:
+        # Determine if using Ollama embedder if not specified
+        if is_ollama_embedder is None:
+            from api.config import is_ollama_embedder as check_ollama
+            is_ollama_embedder = check_ollama()
+
+        if is_ollama_embedder:
             encoding = tiktoken.get_encoding("cl100k_base")
         else:
             encoding = tiktoken.encoding_for_model("text-embedding-3-small")
@@ -117,14 +123,15 @@ def download_repo(repo_url: str, local_path: str, type: str = "github", access_t
 # Alias for backward compatibility
 download_github_repo = download_repo
 
-def read_all_documents(path: str, local_ollama: bool = False, excluded_dirs: List[str] = None, excluded_files: List[str] = None,
+def read_all_documents(path: str, is_ollama_embedder: bool = None, excluded_dirs: List[str] = None, excluded_files: List[str] = None,
                       included_dirs: List[str] = None, included_files: List[str] = None):
     """
     Recursively reads all documents in a directory and its subdirectories.
 
     Args:
         path (str): The root directory path.
-        local_ollama (bool): Whether to use local Ollama for token counting. Default is False.
+        is_ollama_embedder (bool, optional): Whether using Ollama embeddings for token counting.
+                                           If None, will be determined from configuration.
         excluded_dirs (List[str], optional): List of directories to exclude from processing.
             Overrides the default configuration if provided.
         excluded_files (List[str], optional): List of file patterns to exclude from processing.
@@ -282,7 +289,7 @@ def should_process_file(file_path: str, use_inclusion: bool, included_dirs: List
                     )
 
                     # Check token count
-                    token_count = count_tokens(content, local_ollama)
+                    token_count = count_tokens(content, is_ollama_embedder)
                     if token_count > MAX_EMBEDDING_TOKENS * 10:
                         logger.warning(f"Skipping large file {relative_path}: Token count ({token_count}) exceeds limit")
                         continue
@@ -316,7 +323,7 @@ def should_process_file(file_path: str, use_inclusion: bool, included_dirs: List
                     relative_path = os.path.relpath(file_path, path)
 
                     # Check token count
-                    token_count = count_tokens(content, local_ollama)
+                    token_count = count_tokens(content, is_ollama_embedder)
                     if token_count > MAX_EMBEDDING_TOKENS:
                         logger.warning(f"Skipping large file {relative_path}: Token count ({token_count}) exceeds limit")
                         continue
@@ -339,33 +346,43 @@ def should_process_file(file_path: str, use_inclusion: bool, included_dirs: List
     logger.info(f"Found {len(documents)} documents")
     return documents
 
-def prepare_data_pipeline(local_ollama: bool = False):
+def prepare_data_pipeline(is_ollama_embedder: bool = None):
     """
     Creates and returns the data transformation pipeline.
 
     Args:
-        local_ollama (bool): Whether to use local Ollama for embedding (default: False)
+        is_ollama_embedder (bool, optional): Whether to use Ollama for embedding.
+                                           If None, will be determined from configuration.
 
     Returns:
         adal.Sequential: The data transformation pipeline
     """
+    from api.config import get_embedder_config, is_ollama_embedder as check_ollama
+
+    # Determine if using Ollama embedder if not specified
+    if is_ollama_embedder is None:
+        is_ollama_embedder = check_ollama()
+
     splitter = TextSplitter(**configs["text_splitter"])
+    embedder_config = get_embedder_config()
 
-    if local_ollama:
-        # Use Ollama embedder
-        embedder = adal.Embedder(
-            model_client=configs["embedder_ollama"]["model_client"](),
-            model_kwargs=configs["embedder_ollama"]["model_kwargs"],
-        )
+    if not embedder_config:
+        raise ValueError("No embedder configuration found")
+
+    # Create embedder based on configuration
+    embedder = adal.Embedder(
+        model_client=embedder_config["model_client"](),
+        model_kwargs=embedder_config["model_kwargs"],
+    )
+
+    if is_ollama_embedder:
+        # Use Ollama document processor for single-document processing
         embedder_transformer = OllamaDocumentProcessor(embedder=embedder)
     else:
-        # Use OpenAI embedder
-        embedder = adal.Embedder(
-            model_client=configs["embedder"]["model_client"](),
-            model_kwargs=configs["embedder"]["model_kwargs"],
-        )
+        # Use batch processing for other embedders
+        batch_size = embedder_config.get("batch_size", 500)
         embedder_transformer = ToEmbeddings(
-            embedder=embedder, batch_size=configs["embedder"]["batch_size"]
+            embedder=embedder, batch_size=batch_size
         )
 
     data_transformer = adal.Sequential(
@@ -374,18 +391,19 @@ def prepare_data_pipeline(local_ollama: bool = False):
     return data_transformer
 
 def transform_documents_and_save_to_db(
-    documents: List[Document], db_path: str, local_ollama: bool = False
+    documents: List[Document], db_path: str, is_ollama_embedder: bool = None
 ) -> LocalDB:
     """
     Transforms a list of documents and saves them to a local database.
 
     Args:
         documents (list): A list of `Document` objects.
         db_path (str): The path to the local database file.
-        local_ollama (bool): Whether to use local Ollama for embedding (default: False)
+        is_ollama_embedder (bool, optional): Whether to use Ollama for embedding.
+                                           If None, will be determined from configuration.
     """
     # Get the data transformer
-    data_transformer = prepare_data_pipeline(local_ollama)
+    data_transformer = prepare_data_pipeline(is_ollama_embedder)
 
     # Save the documents to a local database
     db = LocalDB()
@@ -642,7 +660,7 @@ def __init__(self):
         self.repo_url_or_path = None
         self.repo_paths = None
 
-    def prepare_database(self, repo_url_or_path: str, type: str = "github", access_token: str = None, local_ollama: bool = False,
+    def prepare_database(self, repo_url_or_path: str, type: str = "github", access_token: str = None, is_ollama_embedder: bool = None,
                        excluded_dirs: List[str] = None, excluded_files: List[str] = None,
                        included_dirs: List[str] = None, included_files: List[str] = None) -> List[Document]:
         """
@@ -651,7 +669,8 @@ def prepare_database(self, repo_url_or_path: str, type: str = "github", access_t
         Args:
             repo_url_or_path (str): The URL or local path of the repository
             access_token (str, optional): Access token for private repositories
-            local_ollama (bool): Whether to use local Ollama for embedding (default: False)
+            is_ollama_embedder (bool, optional): Whether to use Ollama for embedding.
+                                               If None, will be determined from configuration.
             excluded_dirs (List[str], optional): List of directories to exclude from processing
             excluded_files (List[str], optional): List of file patterns to exclude from processing
             included_dirs (List[str], optional): List of directories to include exclusively
@@ -662,7 +681,7 @@ def prepare_database(self, repo_url_or_path: str, type: str = "github", access_t
         """
         self.reset_database()
         self._create_repo(repo_url_or_path, type, access_token)
-        return self.prepare_db_index(local_ollama=local_ollama, excluded_dirs=excluded_dirs, excluded_files=excluded_files,
+        return self.prepare_db_index(is_ollama_embedder=is_ollama_embedder, excluded_dirs=excluded_dirs, excluded_files=excluded_files,
                                    included_dirs=included_dirs, included_files=included_files)
 
     def reset_database(self):
@@ -734,13 +753,14 @@ def _create_repo(self, repo_url_or_path: str, type: str = "github", access_token
             logger.error(f"Failed to create repository structure: {e}")
             raise
 
-    def prepare_db_index(self, local_ollama: bool = False, excluded_dirs: List[str] = None, excluded_files: List[str] = None,
+    def prepare_db_index(self, is_ollama_embedder: bool = None, excluded_dirs: List[str] = None, excluded_files: List[str] = None,
                         included_dirs: List[str] = None, included_files: List[str] = None) -> List[Document]:
         """
         Prepare the indexed database for the repository.
 
         Args:
-            local_ollama (bool): Whether to use local Ollama for embedding (default: False)
+            is_ollama_embedder (bool, optional): Whether to use Ollama for embedding.
+                                               If None, will be determined from configuration.
             excluded_dirs (List[str], optional): List of directories to exclude from processing
             excluded_files (List[str], optional): List of file patterns to exclude from processing
             included_dirs (List[str], optional): List of directories to include exclusively
@@ -766,14 +786,14 @@ def prepare_db_index(self, local_ollama: bool = False, excluded_dirs: List[str]
         logger.info("Creating new database...")
         documents = read_all_documents(
             self.repo_paths["save_repo_dir"],
-            local_ollama=local_ollama,
+            is_ollama_embedder=is_ollama_embedder,
             excluded_dirs=excluded_dirs,
             excluded_files=excluded_files,
             included_dirs=included_dirs,
             included_files=included_files
         )
         self.db = transform_documents_and_save_to_db(
-            documents, self.repo_paths["save_db_file"], local_ollama=local_ollama
+            documents, self.repo_paths["save_db_file"], is_ollama_embedder=is_ollama_embedder
         )
         logger.info(f"Total documents: {len(documents)}")
         transformed_docs = self.db.get_transformed_data(key="split_and_embed")

api/rag.py

@@ -218,15 +218,20 @@ def __init__(self, provider="google", model=None, use_s3: bool = False):  # noqa
 
         self.provider = provider
         self.model = model
-        self.local_ollama = provider == "ollama"
+
+        # Import the helper functions
+        from api.config import get_embedder_config, is_ollama_embedder
+
+        # Determine if we're using Ollama embedder based on configuration
+        self.is_ollama_embedder = is_ollama_embedder()
 
         # Initialize components
         self.memory = Memory()
 
-        if self.local_ollama:
-            embedder_config = configs["embedder_ollama"]
-        else:
-            embedder_config = configs["embedder"]
+        # Get embedder configuration
+        embedder_config = get_embedder_config()
+        if not embedder_config:
+            raise ValueError("No embedder configuration found")
 
         # --- Initialize Embedder ---
         self.embedder = adal.Embedder(
@@ -242,7 +247,9 @@ def single_string_embedder(query):
                     raise ValueError("Ollama embedder only supports a single string")
                 query = query[0]
             return self.embedder(input=query)
-        self.query_embedder = single_string_embedder
+
+        # Use single string embedder for Ollama, regular embedder for others
+        self.query_embedder = single_string_embedder if self.is_ollama_embedder else self.embedder
 
         self.initialize_db_manager()
 
@@ -402,7 +409,7 @@ def prepare_retriever(self, repo_url_or_path: str, type: str = "github", access_
             repo_url_or_path,
             type,
             access_token,
-            local_ollama=self.local_ollama,
+            is_ollama_embedder=self.is_ollama_embedder,
             excluded_dirs=excluded_dirs,
             excluded_files=excluded_files,
             included_dirs=included_dirs,
@@ -419,10 +426,11 @@ def prepare_retriever(self, repo_url_or_path: str, type: str = "github", access_
         logger.info(f"Using {len(self.transformed_docs)} documents with valid embeddings for retrieval")
 
         try:
-            retreive_embedder = self.query_embedder if self.local_ollama else self.embedder
+            # Use the appropriate embedder for retrieval
+            retrieve_embedder = self.query_embedder if self.is_ollama_embedder else self.embedder
             self.retriever = FAISSRetriever(
                 **configs["retriever"],
-                embedder=retreive_embedder,
+                embedder=retrieve_embedder,
                 documents=self.transformed_docs,
                 document_map_func=lambda doc: doc.vector,
             )

src/app/globals.css

@@ -14,6 +14,7 @@
   --card-bg: #fffaf0; /* Slightly warmer than background */
   --highlight: #e8927c; /* Soft coral (Akane) */
   --muted: #a59e8c; /* Soft gray-brown (Nezumi) */
+  --link-color: #7c5aa0; /* Slightly darker purple for links */
 }
 
 html[data-theme='dark'] {
@@ -27,6 +28,7 @@ html[data-theme='dark'] {
   --card-bg: #222222; /* Slightly lighter than background */
   --highlight: #e57373; /* Soft red */
   --muted: #8c8c8c; /* Muted gray */
+  --link-color: #b19cd9; /* Lighter purple for dark mode links */
 }
 
 @theme inline {
@@ -117,3 +119,25 @@ html[data-theme='dark'] .paper-texture {
 .card-japanese:hover {
   box-shadow: 0 4px 12px var(--shadow-color);
 }
+
+/* Line clamp utilities */
+.line-clamp-1 {
+  overflow: hidden;
+  display: -webkit-box;
+  -webkit-box-orient: vertical;
+  -webkit-line-clamp: 1;
+}
+
+.line-clamp-2 {
+  overflow: hidden;
+  display: -webkit-box;
+  -webkit-box-orient: vertical;
+  -webkit-line-clamp: 2;
+}
+
+.line-clamp-3 {
+  overflow: hidden;
+  display: -webkit-box;
+  -webkit-box-orient: vertical;
+  -webkit-line-clamp: 3;
+}