Merge pull request #1325 from TriliumNext/ai-llm-integration

[WIP] AI/LLM integration

Author: eliandoran

db/migrations/0230__vector_embeddings.sql

@@ -0,0 +1,46 @@
+-- Add tables for vector embeddings storage and management
+-- This migration adds embedding support to the main document.db database
+
+-- Store embeddings for notes
+CREATE TABLE IF NOT EXISTS "note_embeddings" (
+    "embedId" TEXT NOT NULL PRIMARY KEY,
+    "noteId" TEXT NOT NULL,
+    "providerId" TEXT NOT NULL,
+    "modelId" TEXT NOT NULL,
+    "dimension" INTEGER NOT NULL,
+    "embedding" BLOB NOT NULL,
+    "version" INTEGER NOT NULL DEFAULT 1,
+    "dateCreated" TEXT NOT NULL,
+    "utcDateCreated" TEXT NOT NULL,
+    "dateModified" TEXT NOT NULL,
+    "utcDateModified" TEXT NOT NULL
+);
+
+CREATE INDEX "IDX_note_embeddings_noteId" ON "note_embeddings" ("noteId");
+CREATE INDEX "IDX_note_embeddings_providerId_modelId" ON "note_embeddings" ("providerId", "modelId");
+
+-- Table to track which notes need embedding updates
+CREATE TABLE IF NOT EXISTS "embedding_queue" (
+    "noteId" TEXT NOT NULL PRIMARY KEY,
+    "operation" TEXT NOT NULL, -- CREATE, UPDATE, DELETE
+    "dateQueued" TEXT NOT NULL,
+    "utcDateQueued" TEXT NOT NULL,
+    "priority" INTEGER NOT NULL DEFAULT 0,
+    "attempts" INTEGER NOT NULL DEFAULT 0,
+    "lastAttempt" TEXT NULL,
+    "error" TEXT NULL,
+    "failed" INTEGER NOT NULL DEFAULT 0,
+    "isProcessing" INTEGER NOT NULL DEFAULT 0
+);
+
+-- Table to store embedding provider configurations
+CREATE TABLE IF NOT EXISTS "embedding_providers" (
+    "providerId" TEXT NOT NULL PRIMARY KEY,
+    "name" TEXT NOT NULL,
+    "priority" INTEGER NOT NULL DEFAULT 0,
+    "config" TEXT NOT NULL, -- JSON config object
+    "dateCreated" TEXT NOT NULL,
+    "utcDateCreated" TEXT NOT NULL,
+    "dateModified" TEXT NOT NULL,
+    "utcDateModified" TEXT NOT NULL
+);

db/schema.sql

@@ -145,3 +145,45 @@ CREATE INDEX IDX_attachments_ownerId_role
 CREATE INDEX IDX_notes_blobId on notes (blobId);
 CREATE INDEX IDX_revisions_blobId on revisions (blobId);
 CREATE INDEX IDX_attachments_blobId on attachments (blobId);
+
+CREATE TABLE IF NOT EXISTS "note_embeddings" (
+    "embedId" TEXT NOT NULL PRIMARY KEY,
+    "noteId" TEXT NOT NULL,
+    "providerId" TEXT NOT NULL,
+    "modelId" TEXT NOT NULL,
+    "dimension" INTEGER NOT NULL,
+    "embedding" BLOB NOT NULL,
+    "version" INTEGER NOT NULL DEFAULT 1,
+    "dateCreated" TEXT NOT NULL,
+    "utcDateCreated" TEXT NOT NULL,
+    "dateModified" TEXT NOT NULL,
+    "utcDateModified" TEXT NOT NULL
+);
+
+CREATE INDEX "IDX_note_embeddings_noteId" ON "note_embeddings" ("noteId");
+CREATE INDEX "IDX_note_embeddings_providerId_modelId" ON "note_embeddings" ("providerId", "modelId");
+
+CREATE TABLE IF NOT EXISTS "embedding_queue" (
+    "noteId" TEXT NOT NULL PRIMARY KEY,
+    "operation" TEXT NOT NULL,
+    "dateQueued" TEXT NOT NULL,
+    "utcDateQueued" TEXT NOT NULL,
+    "priority" INTEGER NOT NULL DEFAULT 0,
+    "attempts" INTEGER NOT NULL DEFAULT 0,
+    "lastAttempt" TEXT NULL,
+    "error" TEXT NULL,
+    "failed" INTEGER NOT NULL DEFAULT 0,
+    "isProcessing" INTEGER NOT NULL DEFAULT 0
+);
+
+CREATE TABLE IF NOT EXISTS "embedding_providers" (
+    "providerId" TEXT NOT NULL PRIMARY KEY,
+    "name" TEXT NOT NULL,
+    "isEnabled" INTEGER NOT NULL DEFAULT 0,
+    "priority" INTEGER NOT NULL DEFAULT 0,
+    "config" TEXT NOT NULL,
+    "dateCreated" TEXT NOT NULL,
+    "utcDateCreated" TEXT NOT NULL,
+    "dateModified" TEXT NOT NULL,
+    "utcDateModified" TEXT NOT NULL
+);

docs/Developer Guide/!!!meta.json

@@ -1,6 +1,6 @@
 {
     "formatVersion": 2,
-    "appVersion": "0.92.7",
+    "appVersion": "0.93.0",
     "files": [
         {
             "isClone": false,

docs/Release Notes/!!!meta.json

@@ -1,6 +1,6 @@
 {
     "formatVersion": 2,
-    "appVersion": "0.92.7",
+    "appVersion": "0.93.0",
     "files": [
         {
             "isClone": false,

docs/Release Notes/Release Notes/v0.93.0.md

@@ -1,5 +1,4 @@
 # v0.93.0
-
 ## 🐞 Bugfixes
 
 *   Calendar does not hide when clicking on a note by @JYC333