Add idempotent indexing with file hash tracking to skip unchanged files, update modified files, and support force reindex, including updated mock helpers and test coverage.

Author: cpetersen

lib/ragnar/cli.rb

@@ -43,6 +43,7 @@ class CLI < Thor
     option :chunk_size, type: :numeric, desc: "Chunk size in tokens (default from config)"
     option :chunk_overlap, type: :numeric, desc: "Chunk overlap in tokens (default from config)"
     option :model, type: :string, desc: "Embedding model to use (default from config)"
+    option :force, type: :boolean, default: false, aliases: "-f", desc: "Force reindex all files even if unchanged"
     def index(path)
       # Expand user paths (handle ~ in user input)
       expanded_path = File.expand_path(path)
@@ -76,11 +77,39 @@ def index(path)
       )
 
       begin
-        stats = indexer.index_path(expanded_path)
+        stats = indexer.index_path(expanded_path, force: options[:force])
+        
         say "\nIndexing complete!", :green
-        say "Files processed: #{stats[:files_processed]}"
-        say "Chunks created: #{stats[:chunks_created]}"
-        say "Errors: #{stats[:errors]}" if stats[:errors] > 0
+        
+        # Display detailed stats
+        if stats[:files_skipped] > 0
+          say "  Files skipped (unchanged): #{stats[:files_skipped]}", :cyan
+        end
+        
+        if stats[:files_processed] > 0
+          say "  New files indexed: #{stats[:files_processed]}", :green
+        end
+        
+        if stats[:files_updated] > 0
+          say "  Files updated: #{stats[:files_updated]}", :yellow
+          if stats[:chunks_deleted] > 0 || stats[:chunks_created] > 0
+            say "    Chunks: -#{stats[:chunks_deleted]} removed, +#{stats[:chunks_created]} added", :yellow
+          end
+        elsif stats[:chunks_created] > 0
+          say "  Chunks created: #{stats[:chunks_created]}", :green
+        end
+        
+        if stats[:errors] > 0
+          say "  Errors: #{stats[:errors]}", :red
+        end
+        
+        # Summary
+        total_files = stats[:files_processed] + stats[:files_skipped] + stats[:files_updated]
+        say "\nSummary: #{total_files} files scanned", :cyan
+        
+        if options[:force]
+          say "  (Force reindex was enabled)", :yellow
+        end
       rescue => e
         say "Error during indexing: #{e.message}", :red
         exit 1

lib/ragnar/database.rb

@@ -1,3 +1,5 @@
+require 'fileutils'
+
 module Ragnar
   class Database
     attr_reader :db_path, :table_name
@@ -6,6 +8,7 @@ def initialize(db_path, table_name: "documents")
       @db_path = db_path
       @table_name = table_name
       @dataset_cache = nil  # Cache to prevent file descriptor leaks
+      @file_metadata_cache = nil  # Cache for file metadata dataset
       ensure_database_exists
     end
 
@@ -251,6 +254,143 @@ def get_all_documents_with_embeddings(limit: nil)
       all_docs.select { |doc| doc[:embedding] && !doc[:embedding].empty? }
     end
 
+    # File metadata tracking methods for idempotent indexing
+    
+    # Get metadata for a specific file
+    # @param file_path [String] The path of the file
+    # @return [Hash, nil] File metadata or nil if not found
+    def get_file_metadata(file_path)
+      return nil unless file_metadata_exists?
+      
+      metadata_dataset = cached_file_metadata_dataset
+      return nil unless metadata_dataset
+      
+      metadata_dataset.to_a.find { |row| row[:file_path] == file_path }
+    end
+    
+    # Store or update file metadata
+    # @param file_path [String] The path of the file
+    # @param file_hash [String] Hash of the file contents
+    # @param chunk_count [Integer] Number of chunks created
+    def upsert_file_metadata(file_path, file_hash, chunk_count)
+      metadata = {
+        file_path: file_path,
+        file_hash: file_hash,
+        chunk_count: chunk_count,
+        indexed_at: Time.now.iso8601
+      }
+      
+      # Define schema for file metadata table
+      schema = {
+        file_path: :string,
+        file_hash: :string,
+        chunk_count: :int64,
+        indexed_at: :string
+      }
+      
+      # Clear cache before modifying
+      clear_file_metadata_cache
+      
+      if file_metadata_exists?
+        # Update existing or add new
+        metadata_dataset = Lancelot::Dataset.open(file_metadata_path)
+        existing_data = metadata_dataset.to_a.reject { |row| row[:file_path] == file_path }
+        existing_data << metadata
+        
+        # Recreate dataset with updated data
+        FileUtils.rm_rf(file_metadata_path) if File.exist?(file_metadata_path)
+        new_dataset = Lancelot::Dataset.open_or_create(file_metadata_path, schema: schema)
+        new_dataset.add_documents(existing_data) unless existing_data.empty?
+      else
+        # Create new metadata dataset
+        dataset = Lancelot::Dataset.open_or_create(file_metadata_path, schema: schema)
+        dataset.add_documents([metadata])
+      end
+      
+      # Clear cache after modification
+      clear_file_metadata_cache
+    end
+    
+    # Delete all chunks associated with a file
+    # @param file_path [String] The path of the file
+    # @return [Integer] Number of chunks deleted
+    def delete_chunks_by_file(file_path)
+      return 0 unless dataset_exists?
+      
+      dataset = cached_dataset
+      return 0 unless dataset
+      
+      # Get all documents and filter out the ones from this file
+      all_docs = dataset.to_a
+      chunks_to_keep = all_docs.reject { |doc| doc[:file_path] == file_path }
+      deleted_count = all_docs.size - chunks_to_keep.size
+      
+      if deleted_count > 0
+        # Clear cache before modifying
+        clear_dataset_cache
+        
+        # Recreate the dataset without the deleted chunks
+        if chunks_to_keep.empty?
+          # If no chunks left, just remove the dataset
+          FileUtils.rm_rf(@db_path) if File.exist?(@db_path)
+        else
+          # Get schema from first remaining document
+          first_doc = chunks_to_keep.first
+          embedding_size = first_doc[:embedding]&.size || 768
+          reduced_size = first_doc[:reduced_embedding]&.size
+          
+          schema = {
+            id: :string,
+            chunk_text: :string,
+            file_path: :string,
+            chunk_index: :int64,
+            embedding: { type: "vector", dimension: embedding_size },
+            metadata: :string
+          }
+          
+          # Add reduced_embedding if present
+          if reduced_size
+            schema[:reduced_embedding] = { type: "vector", dimension: reduced_size }
+          end
+          
+          # Recreate dataset
+          FileUtils.rm_rf(@db_path) if File.exist?(@db_path)
+          new_dataset = Lancelot::Dataset.open_or_create(@db_path, schema: schema)
+          new_dataset.add_documents(chunks_to_keep)
+        end
+        
+        # Clear cache after modification
+        clear_dataset_cache
+      end
+      
+      deleted_count
+    end
+    
+    # Create the file metadata table
+    def create_file_metadata_table
+      schema = {
+        file_path: :string,
+        file_hash: :string,
+        chunk_count: :int64,
+        indexed_at: :string
+      }
+      
+      Lancelot::Dataset.open_or_create(file_metadata_path, schema: schema)
+      clear_file_metadata_cache
+    end
+    
+    # Check if a table exists (for migration purposes)
+    # @param table_name [String] Name of the table to check
+    # @return [Boolean] true if table exists
+    def table_exists?(table_name)
+      case table_name
+      when "file_metadata"
+        file_metadata_exists?
+      else
+        dataset_exists?
+      end
+    end
+    
     def full_text_search(query, limit: 10)
       return [] unless dataset_exists?
 
@@ -330,8 +470,30 @@ def ensure_database_exists
       # Don't create directory - Lance will handle this
     end
 
-    def table_exists?
-      dataset_exists?
+    # Path to the file metadata dataset
+    def file_metadata_path
+      File.join(File.dirname(@db_path), "file_metadata")
+    end
+    
+    # Check if file metadata dataset exists
+    def file_metadata_exists?
+      File.exist?(file_metadata_path)
+    end
+    
+    # Cached file metadata dataset accessor
+    def cached_file_metadata_dataset
+      return nil unless File.exist?(file_metadata_path)
+      
+      @file_metadata_cache ||= begin
+        Lancelot::Dataset.open(file_metadata_path)
+      rescue => e
+        nil
+      end
+    end
+    
+    # Clear the cached file metadata dataset
+    def clear_file_metadata_cache
+      @file_metadata_cache = nil
     end
 
     # Cached dataset accessor to prevent file descriptor leaks