Merge pull request #48 from rryam/feature/nl-contextual-embedding

Add NaturalLanguage framework support via VecturaNLKit

Author: rudrankriyam

Package.swift

@@ -21,6 +21,10 @@ let package = Package(
       name: "VecturaMLXKit",
       targets: ["VecturaMLXKit"]
     ),
+    .library(
+      name: "VecturaNLKit",
+      targets: ["VecturaNLKit"]
+    ),
     .executable(
       name: "vectura-cli",
       targets: ["VecturaCLI"]
@@ -53,6 +57,12 @@ let package = Package(
         .product(name: "MLXEmbedders", package: "mlx-swift-lm"),
       ]
     ),
+    .target(
+      name: "VecturaNLKit",
+      dependencies: [
+        "VecturaKit"
+      ]
+    ),
     .executableTarget(
       name: "VecturaCLI",
       dependencies: [
@@ -76,6 +86,10 @@ let package = Package(
       name: "TestMLXExamples",
       dependencies: ["VecturaMLXKit"]
     ),
+    .executableTarget(
+      name: "TestNLExamples",
+      dependencies: ["VecturaNLKit"]
+    ),
     .testTarget(
       name: "VecturaKitTests",
       dependencies: ["VecturaKit"]
@@ -84,6 +98,10 @@ let package = Package(
       name: "VecturaMLXKitTests",
       dependencies: ["VecturaMLXKit"]
     ),
+    .testTarget(
+      name: "VecturaNLKitTests",
+      dependencies: ["VecturaNLKit"]
+    ),
     .testTarget(
       name: "PerformanceTests",
       dependencies: ["VecturaKit"],

README.md

@@ -1,10 +1,10 @@
 # VecturaKit
 
-VecturaKit is a Swift-based vector database designed for on-device apps through local vector storage and retrieval. 
+VecturaKit is a Swift-based vector database designed for on-device apps through local vector storage and retrieval.
 
 Inspired by [Dripfarm's SVDB](https://github.com/Dripfarm/SVDB), **VecturaKit** uses `MLTensor` and [`swift-embeddings`](https://github.com/jkrukowski/swift-embeddings) for generating and managing embeddings. It features **Model2Vec** support with the 32M parameter model as default for fast static embeddings.
 
-The framework offers `VecturaKit` as the core vector database with pluggable embedding providers. Use `SwiftEmbedder` for `swift-embeddings` integration or `MLXEmbedder` for Apple's MLX framework acceleration.
+The framework offers `VecturaKit` as the core vector database with pluggable embedding providers. Use `SwiftEmbedder` for `swift-embeddings` integration, `MLXEmbedder` for Apple's MLX framework acceleration, or `NLContextualEmbedder` for Apple's NaturalLanguage framework with zero external dependencies.
 
 It also includes CLI tools (`vectura-cli` and `vectura-mlx-cli`) for easily trying out the package.
 
@@ -55,6 +55,12 @@ Explore the following books to understand more about AI and iOS development:
   - [Add Documents](#add-documents-1)
   - [Search Documents](#search-documents-1)
   - [Document Management](#document-management-1)
+- [NaturalLanguage Integration](#naturallanguage-integration)
+  - [Import NaturalLanguage Support](#import-naturallanguage-support)
+  - [Initialize Database with NLContextualEmbedding](#initialize-database-with-nlcontextualembedding)
+  - [Add Documents](#nl-add-documents)
+  - [Search Documents](#nl-search-documents)
+  - [Document Management](#nl-document-management)
 - [Command Line Interface](#command-line-interface)
   - [Swift CLI Tool (`vectura-cli`)](#swift-cli-tool-vectura-cli)
   - [MLX CLI Tool (`vectura-mlx-cli`)](#mlx-cli-tool-vectura-mlx-cli)
@@ -76,6 +82,7 @@ Explore the following books to understand more about AI and iOS development:
 -   **Custom Storage Provider:** Implements custom storage backends (SQLite, Core Data, cloud storage) by conforming to the `VecturaStorage` protocol.
 -   **Memory Management Strategies:** Choose between automatic, full-memory, or indexed modes to optimize performance for datasets ranging from thousands to millions of documents. [Learn more](Docs/INDEXED_STORAGE_GUIDE.md)
 -   **MLX Support:** Uses Apple's MLX framework for accelerated embedding generation through `MLXEmbedder`.
+-   **NaturalLanguage Support:** Uses Apple's NaturalLanguage framework for contextual embeddings with zero external dependencies through `NLContextualEmbedder`.
 -   **CLI Tools:** Includes `vectura-cli` (Swift embeddings) and `vectura-mlx-cli` (MLX embeddings) for database management and testing.
 
 ## Supported Platforms
@@ -106,6 +113,8 @@ VecturaKit uses the following Swift packages:
 -   [swift-argument-parser](https://github.com/apple/swift-argument-parser): Used for creating the command-line interface.
 -   [mlx-swift-examples](https://github.com/ml-explore/mlx-swift-examples): Provides MLX-based embeddings and vector search capabilities, specifically for `VecturaMLXKit`.
 
+**Note:** `VecturaNLKit` has no external dependencies beyond Apple's native NaturalLanguage framework.
+
 ## Quick Start
 
 Get up and running with VecturaKit in minutes. Here is an example of adding and searching documents:
@@ -480,6 +489,121 @@ Reset database:
 try await vectorDB.reset()
 ```
 
+## NaturalLanguage Integration
+
+VecturaKit supports Apple's NaturalLanguage framework through the `NLContextualEmbedder` for contextual embeddings with zero external dependencies.
+
+### Import NaturalLanguage Support
+
+```swift
+import VecturaKit
+import VecturaNLKit
+```
+
+### Initialize Database with NLContextualEmbedding
+
+```swift
+let config = VecturaConfig(
+  name: "my-nl-vector-db",
+  dimension: nil  // Auto-detect dimension from NL embedder
+)
+
+// Create NLContextualEmbedder
+let embedder = try await NLContextualEmbedder(
+  language: .english
+)
+let vectorDB = try await VecturaKit(config: config, embedder: embedder)
+```
+
+**Available Options:**
+
+```swift
+// Initialize with specific language
+let embedder = try await NLContextualEmbedder(
+  language: .spanish
+)
+
+// Get model information
+let modelInfo = await embedder.modelInfo
+print("Language: \(modelInfo.language)")
+if let dimension = modelInfo.dimension {
+  print("Dimension: \(dimension)")
+} else {
+  print("Dimension: Not yet determined")
+}
+```
+
+### <a name="nl-add-documents"></a>Add Documents
+
+```swift
+let texts = [
+  "Natural language understanding is fascinating",
+  "Swift makes iOS development enjoyable",
+  "Machine learning on device preserves privacy"
+]
+let documentIds = try await vectorDB.addDocuments(texts: texts)
+```
+
+### <a name="nl-search-documents"></a>Search Documents
+
+```swift
+let results = try await vectorDB.search(
+    query: "iOS programming",
+    numResults: 5,      // Optional
+    threshold: 0.7     // Optional
+)
+
+for result in results {
+    print("Document ID: \(result.id)")
+    print("Text: \(result.text)")
+    print("Similarity Score: \(result.score)")
+    print("Created At: \(result.createdAt)")
+}
+```
+
+### <a name="nl-document-management"></a>Document Management
+
+Update document:
+
+```swift
+try await vectorDB.updateDocument(
+     id: documentId,
+     newText: "Updated text"
+ )
+```
+
+Delete documents:
+
+```swift
+try await vectorDB.deleteDocuments(ids: [documentId1, documentId2])
+```
+
+Reset database:
+
+```swift
+try await vectorDB.reset()
+```
+
+**Key Features:**
+
+- **Zero External Dependencies:** Uses only Apple's native NaturalLanguage framework
+- **Contextual Embeddings:** Considers surrounding context for more accurate semantic understanding
+- **Privacy-First:** All processing happens on-device
+- **Language Support:** Supports multiple languages (English, Spanish, French, German, Italian, Portuguese, and more)
+- **Auto-Detection:** Automatically detects embedding dimensions
+
+**Performance Characteristics:**
+
+- **Speed:** Moderate (slower than Model2Vec, comparable to MLX)
+- **Accuracy:** High contextual understanding for supported languages
+- **Memory:** Efficient on-device processing
+- **Use Cases:** Ideal for apps requiring semantic search without external dependencies
+
+**Platform Requirements:**
+
+- iOS 17.0+ / macOS 14.0+ / tvOS 17.0+ / visionOS 1.0+ / watchOS 10.0+
+- NaturalLanguage framework (included with OS)
+
 ## Command Line Interface
 
 VecturaKit includes command-line tools for database management with different embedding backends.

Sources/TestNLExamples/TestNLExamples.swift

@@ -0,0 +1,176 @@
+// Test script for VecturaNLKit README examples
+import Foundation
+import VecturaKit
+import VecturaNLKit
+
+enum ExampleError: Error, CustomStringConvertible {
+  case noDocumentsToUpdate
+
+  var description: String {
+    switch self {
+    case .noDocumentsToUpdate:
+      return "No documents available to update"
+    }
+  }
+}
+
+@main
+struct TestNLExamples {
+  static func main() async throws {
+    try await initializeEmbedder()
+    let embedder = try await NLContextualEmbedder(language: .english)
+    let vectorDB = try await initializeDatabase(embedder: embedder)
+    let documentIds = try await addDocuments(to: vectorDB)
+    try await searchDocuments(in: vectorDB)
+    try await testContextualUnderstanding(in: vectorDB)
+    try await manageDocuments(in: vectorDB, documentIds: documentIds)
+    try await testEmbeddings(embedder: embedder)
+    try await resetDatabase(vectorDB)
+
+    debugPrint("\n✅ All VecturaNLKit examples completed successfully!")
+  }
+
+  private static func initializeEmbedder() async throws {
+    debugPrint("1. Initialize NLContextualEmbedder")
+
+    let embedder = try await NLContextualEmbedder(language: .english)
+    let dimension = try await embedder.dimension
+    debugPrint("NLContextualEmbedder initialized successfully")
+    debugPrint("Embedding dimension: \(dimension)")
+
+    let modelInfo = await embedder.modelInfo
+    debugPrint("Model language: \(modelInfo.language.rawValue)")
+    if let dimension = modelInfo.dimension {
+      debugPrint("Model dimension: \(dimension)")
+    }
+  }
+
+  private static func initializeDatabase(embedder: NLContextualEmbedder) async throws -> VecturaKit {
+    debugPrint("\n2. Initialize Database")
+
+    let config = try VecturaConfig(name: "test-nl-vector-db")
+    let vectorDB = try await VecturaKit(
+      config: config,
+      embedder: embedder
+    )
+    debugPrint("NL Database initialized successfully")
+    debugPrint("Document count: \(try await vectorDB.documentCount)")
+
+    return vectorDB
+  }
+
+  private static func addDocuments(to vectorDB: VecturaKit) async throws -> [UUID] {
+    debugPrint("\n3. Add Documents")
+
+    let texts = [
+      "Natural language understanding is fascinating",
+      "Swift makes iOS development enjoyable",
+      "Machine learning on device preserves privacy",
+      "Vector databases enable semantic search"
+    ]
+    let documentIds = try await vectorDB.addDocuments(texts: texts)
+    debugPrint("Documents added with IDs: \(documentIds)")
+    debugPrint("Total document count: \(try await vectorDB.documentCount)")
+
+    return documentIds
+  }
+
+  private static func searchDocuments(in vectorDB: VecturaKit) async throws {
+    debugPrint("\n4. Search Documents")
+
+    let results = try await vectorDB.search(
+      query: "iOS programming",
+      numResults: 5,      // Optional
+      threshold: 0.7      // Optional
+    )
+
+    debugPrint("Search found \(results.count) results:")
+    for result in results {
+      debugPrint("ID: \(result.id)")
+      debugPrint("Text: \(result.text)")
+      debugPrint("Score: \(result.score)")
+      debugPrint("Created At: \(result.createdAt)")
+      debugPrint("---")
+    }
+  }
+
+  private static func testContextualUnderstanding(in vectorDB: VecturaKit) async throws {
+    debugPrint("\n5. Test Contextual Understanding")
+
+    let semanticResults = try await vectorDB.search(
+      query: "building apps for Apple platforms",
+      numResults: 3,
+      threshold: 0.6
+    )
+
+    debugPrint("Semantic search found \(semanticResults.count) results:")
+    for result in semanticResults {
+      debugPrint("Text: \(result.text)")
+      debugPrint("Score: \(result.score)")
+      debugPrint("---")
+    }
+  }
+
+  private static func manageDocuments(in vectorDB: VecturaKit, documentIds: [UUID]) async throws {
+    debugPrint("\n6. Document Management")
+
+    guard let documentToUpdate = documentIds.first else {
+      throw ExampleError.noDocumentsToUpdate
+    }
+
+    debugPrint("Updating document...")
+    try await vectorDB.updateDocument(
+      id: documentToUpdate,
+      newText: "Apple's frameworks enable powerful on-device AI"
+    )
+    debugPrint("Document updated")
+
+    // Verify update by searching
+    let updatedResults = try await vectorDB.search(
+      query: "on-device AI",
+      threshold: 0.6
+    )
+    debugPrint("Verification: Found \(updatedResults.count) documents related to 'on-device AI'")
+    if let first = updatedResults.first {
+      debugPrint("Top result: \(first.text)")
+    }
+
+    debugPrint("\nDeleting documents...")
+    let idsToDelete = documentIds.count >= 2
+      ? [documentToUpdate, documentIds[1]]
+      : [documentToUpdate]
+    try await vectorDB.deleteDocuments(ids: idsToDelete)
+    debugPrint("Documents deleted")
+    debugPrint("Document count after deletion: \(try await vectorDB.documentCount)")
+  }
+
+  private static func testEmbeddings(embedder: NLContextualEmbedder) async throws {
+    debugPrint("\n7. Test Single Embedding")
+
+    let singleText = "Testing NLContextualEmbedding"
+    let singleEmbedding = try await embedder.embed(text: singleText)
+    debugPrint("Generated embedding for: '\(singleText)'")
+    debugPrint("Embedding length: \(singleEmbedding.count)")
+    debugPrint("First 5 values: \(Array(singleEmbedding.prefix(5)))")
+
+    debugPrint("\n8. Test Batch Embedding")
+
+    let batchTexts = [
+      "First test",
+      "Second test",
+      "Third test"
+    ]
+    let batchEmbeddings = try await embedder.embed(texts: batchTexts)
+    debugPrint("Generated \(batchEmbeddings.count) embeddings")
+    for (index, embedding) in batchEmbeddings.enumerated() {
+      debugPrint("Embedding \(index + 1): length = \(embedding.count)")
+    }
+  }
+
+  private static func resetDatabase(_ vectorDB: VecturaKit) async throws {
+    debugPrint("\nResetting database...")
+    try await vectorDB.reset()
+    debugPrint("Database reset")
+    debugPrint("Document count after reset: \(try await vectorDB.documentCount)")
+  }
+}