Add Qdrant as vector database

.github/workflows/build-and-release.yml

@@ -173,6 +173,9 @@ jobs:
           pdfium_version=$(sed -n '11p' metadata.txt)
           pdfium_version=$(echo $pdfium_version | cut -d'.' -f3)
 
+          # Next line is the Qdrant version:
+          qdrant_version="v$(sed -n '12p' metadata.txt)"
+
           # Write the metadata to the environment:
           echo "APP_VERSION=${app_version}" >> $GITHUB_ENV
           echo "FORMATTED_APP_VERSION=${formatted_app_version}" >> $GITHUB_ENV
@@ -185,6 +188,7 @@ jobs:
           echo "TAURI_VERSION=${tauri_version}" >> $GITHUB_ENV
           echo "ARCHITECTURE=${{ matrix.dotnet_runtime }}" >> $GITHUB_ENV
           echo "PDFIUM_VERSION=${pdfium_version}" >> $GITHUB_ENV
+          echo "QDRANT_VERSION=${qdrant_version}" >> $GITHUB_ENV
 
           # Log the metadata:
           echo "App version: '${formatted_app_version}'"
@@ -197,6 +201,7 @@ jobs:
           echo "Tauri version: '${tauri_version}'"
           echo "Architecture: '${{ matrix.dotnet_runtime }}'"
           echo "PDFium version: '${pdfium_version}'"
+          echo "Qdrant version: '${qdrant_version}'"
 
       - name: Read and format metadata (Windows)
         if: matrix.platform == 'windows-latest'
@@ -241,6 +246,9 @@ jobs:
           $pdfium_version = $metadata[10]
           $pdfium_version = $pdfium_version.Split('.')[2]
 
+          # Next line is the necessary Qdrant version:
+          $qdrant_version = "v$metadata[12]"
+
           # Write the metadata to the environment:
           Write-Output "APP_VERSION=${app_version}" >> $env:GITHUB_ENV
           Write-Output "FORMATTED_APP_VERSION=${formatted_app_version}" >> $env:GITHUB_ENV
@@ -252,6 +260,7 @@ jobs:
           Write-Output "MUD_BLAZOR_VERSION=${mud_blazor_version}" >> $env:GITHUB_ENV
           Write-Output "ARCHITECTURE=${{ matrix.dotnet_runtime }}" >> $env:GITHUB_ENV
           Write-Output "PDFIUM_VERSION=${pdfium_version}" >> $env:GITHUB_ENV
+          Write-Output "QDRANT_VERSION=${qdrant_version}" >> $env:GITHUB_ENV
 
           # Log the metadata:
           Write-Output "App version: '${formatted_app_version}'"
@@ -264,6 +273,7 @@ jobs:
           Write-Output "Tauri version: '${tauri_version}'"
           Write-Output "Architecture: '${{ matrix.dotnet_runtime }}'"
           Write-Output "PDFium version: '${pdfium_version}'"
+          Write-Output "Qdrant version: '${qdrant_version}'"
 
       - name: Setup .NET
         uses: actions/setup-dotnet@v4
@@ -334,7 +344,7 @@ jobs:
           echo "Cleaning up ..."
           rm -fr "$TMP"
 
-      - name: Install PDFium (Windows)
+      - name: Deploy PDFium (Windows)
         if: matrix.platform == 'windows-latest'
         env:
           PDFIUM_VERSION: ${{ env.PDFIUM_VERSION }}
@@ -385,6 +395,126 @@ jobs:
           Write-Host "Cleaning up ..."
           Remove-Item $ARCHIVE -Force -ErrorAction SilentlyContinue
 
+          # Try to remove the temporary directory, but ignore errors if files are still in use
+          try {
+            Remove-Item $TMP -Recurse -Force -ErrorAction Stop
+            Write-Host "Successfully cleaned up temporary directory: $TMP"
+          } catch {
+            Write-Warning "Could not fully clean up temporary directory: $TMP. This is usually harmless as Windows will clean it up later. Error: $($_.Exception.Message)"
+          }
+      - name: Deploy Qdrant (Unix)
+        if: matrix.platform != 'windows-latest'
+        env:
+          QDRANT_VERSION: ${{ env.QDRANT_VERSION }}
+          DOTNET_RUNTIME: ${{ matrix.dotnet_runtime }}
+        run: |
+          set -e
+
+          # Target directory:
+          TDB_DIR="runtime/resources/databases/qdrant"
+          mkdir -p "$TDB_DIR"
+
+          case "${DOTNET_RUNTIME}" in
+            linux-x64)
+              QDRANT_FILE="x86_64-unknown-linux-gnu.tar.gz"
+              DB_SOURCE="qdrant"
+              DB_TARGET="qdrant"
+              ;;
+            linux-arm64)
+              QDRANT_FILE="aarch64-unknown-linux-musl.tar.gz"
+              DB_SOURCE="qdrant"
+              DB_TARGET="qdrant"
+              ;;
+            osx-x64)
+              QDRANT_FILE="x86_64-apple-darwin.tar.gz"
+              DB_SOURCE="qdrant"
+              DB_TARGET="qdrant"
+              ;;
+            osx-arm64)
+              QDRANT_FILE="aarch64-apple-darwin.tar.gz"
+              DB_SOURCE="qdrant"
+              DB_TARGET="qdrant"
+              ;;
+            *)
+              echo "Unknown platform: ${DOTNET_RUNTIME}"
+              exit 1
+              ;;
+          esac
+
+          QDRANT_URL="https://github.com/qdrant/qdrant/releases/download/v${QDRANT_VERSION}/qdrant-{QDRANT_FILE}"
+
+          echo "Download Qdrant $QDRANT_URL ..."
+          TMP=$(mktemp -d)
+          ARCHIVE="${TMP}/qdrant.tgz"
+
+          curl -fsSL -o "$ARCHIVE" "$QDRANT_URL"
+
+          echo "Extracting Qdrant ..."
+          tar xzf "$ARCHIVE" -C "$TMP"
+          SRC="${TMP}/${DB_SOURCE}"
+
+          if [ ! -f "$SRC" ]; then
+            echo "Was not able to find Qdrant source: $SRC"
+            exit 1
+          fi
+
+          echo "Copy Qdrant from ${DB_TARGET} to ${TDB_DIR}/"
+          cp -f "$SRC" "$TDB_DIR/$DB_TARGET"
+
+          echo "Cleaning up ..."
+          rm -fr "$TMP"
+
+      - name: Deploy Qdrant (Windows)
+        if: matrix.platform == 'windows-latest'
+        env:
+          QDRANT_VERSION: ${{ env.QDRANT_VERSION }}
+          DOTNET_RUNTIME: ${{ matrix.dotnet_runtime }}
+        run: |
+          $TDB_DIR = "runtime\resources\databases\qdrant"
+          New-Item -ItemType Directory -Force -Path $TDB_DIR | Out-Null
+
+          switch ($env:DOTNET_RUNTIME) {
+            "win-x64" {
+              $QDRANT_FILE = "x86_64-pc-windows-msvc.zip"
+              $DB_SOURCE = "qdrant.exe"
+              $DB_TARGET = "qdrant.exe"
+            }
+            "win-arm64" {
+              $QDRANT_FILE = "x86_64-pc-windows-msvc.zip"
+              $DB_SOURCE = "qdrant.exe"
+              $DB_TARGET = "qdrant.exe""
+            }
+            default {
+              Write-Error "Unknown platform: $($env:DOTNET_RUNTIME)"
+              exit 1
+            }
+          }
+
+          QDRANT_URL="https://github.com/qdrant/qdrant/releases/download/v${QDRANT_VERSION}/qdrant-{QDRANT_FILE}"
+          Write-Host "Download $QDRANT_URL ..."
+
+          # Create a unique temporary directory (not just a file)
+          $TMP = Join-Path ([System.IO.Path]::GetTempPath()) ([System.IO.Path]::GetRandomFileName())
+          New-Item -ItemType Directory -Path $TMP -Force | Out-Null
+          $ARCHIVE = Join-Path $TMP "qdrant.tgz"
+
+          Invoke-WebRequest -Uri $QDRANT_URL -OutFile $ARCHIVE
+
+          Write-Host "Extracting Qdrant ..."
+          tar -xzf $ARCHIVE -C $TMP
+
+          $SRC = Join-Path $TMP $DB_SOURCE
+          if (!(Test-Path $SRC)) {
+            Write-Error "Cannot find Qdrant source: $SRC"
+            exit 1
+          }
+
+          $DEST = Join-Path $TDB_DIR $DB_TARGET
+          Copy-Item -Path $SRC -Destination $DEST -Force
+
+          Write-Host "Cleaning up ..."
+          Remove-Item $ARCHIVE -Force -ErrorAction SilentlyContinue
+
           # Try to remove the temporary directory, but ignore errors if files are still in use
           try {
             Remove-Item $TMP -Recurse -Force -ErrorAction Stop

.gitignore

@@ -6,6 +6,13 @@ libpdfium.dylib
 libpdfium.so
 libpdfium.dll
 
+# Ignore qdrant database:
+qdrant-aarch64-apple-darwin
+qdrant-x86_64-apple-darwin
+qdrant-aarch64-unknown-linux-gnu
+qdrant-x86_64-unknown-linux-gnu
+qdrant-x86_64-pc-windows-msvc.exe
+
 # User-specific files
 *.rsuser
 *.suo

README.md

@@ -32,7 +32,7 @@ Since November 2024: Work on RAG (integration of your data and files) has begun.
 - [x] ~~App: Implement dialog for checking & handling [pandoc](https://pandoc.org/) installation ([PR #393](https://github.com/MindWorkAI/AI-Studio/pull/393), [PR #487](https://github.com/MindWorkAI/AI-Studio/pull/487))~~
 - [ ] App: Implement external embedding providers
 - [ ] App: Implement the process to vectorize one local file using embeddings
-- [ ] Runtime: Integration of the vector database [LanceDB](https://github.com/lancedb/lancedb)
+- [ ] Runtime: Integration of the vector database [Qdrant](https://github.com/qdrant/qdrant)
 - [ ] App: Implement the continuous process of vectorizing data
 - [x] ~~App: Define a common retrieval context interface for the integration of RAG processes in chats (PR [#281](https://github.com/MindWorkAI/AI-Studio/pull/281), [#284](https://github.com/MindWorkAI/AI-Studio/pull/284), [#286](https://github.com/MindWorkAI/AI-Studio/pull/286), [#287](https://github.com/MindWorkAI/AI-Studio/pull/287))~~
 - [x] ~~App: Define a common augmentation interface for the integration of RAG processes in chats (PR [#288](https://github.com/MindWorkAI/AI-Studio/pull/288), [#289](https://github.com/MindWorkAI/AI-Studio/pull/289))~~

app/Build/Commands/Database.cs

@@ -0,0 +1,3 @@
+namespace Build.Commands;
+
+public record Database(string Path, string Filename);

app/Build/Commands/Qdrant.cs

@@ -0,0 +1,120 @@
+using System.Diagnostics.Eventing.Reader;
+using System.Formats.Tar;
+using System.IO.Compression;
+
+using SharedTools;
+
+namespace Build.Commands;
+
+public static class Qdrant
+{
+    public static async Task InstallAsync(RID rid, string version)
+    {
+        Console.Write($"- Installing Qdrant {version} for {rid.ToUserFriendlyName()} ...");
+
+        var cwd = Environment.GetRustRuntimeDirectory();
+        var qdrantTmpDownloadPath = Path.GetTempFileName();
+        var qdrantTmpExtractPath = Directory.CreateTempSubdirectory();
+        var qdrantUrl = GetQdrantDownloadUrl(rid, version);
+
+        //
+        // Download the file:
+        //
+        Console.Write(" downloading ...");
+        using (var client = new HttpClient())
+        {
+            var response = await client.GetAsync(qdrantUrl);
+            if (!response.IsSuccessStatusCode)
+            {
+                Console.WriteLine($" failed to download Qdrant {version} for {rid.ToUserFriendlyName()} from {qdrantUrl}");
+                return;
+            }
+
+            await using var fileStream = File.Create(qdrantTmpDownloadPath);
+            await response.Content.CopyToAsync(fileStream);
+        }
+
+        //
+        // Extract the downloaded file:
+        //
+        Console.Write(" extracting ...");
+        await using(var zStream = File.Open(qdrantTmpDownloadPath, FileMode.Open, FileAccess.Read, FileShare.Read))
+        {
+            if (rid == RID.WIN_X64)
+            {
+                using var archive = new ZipArchive(zStream, ZipArchiveMode.Read);
+                archive.ExtractToDirectory(qdrantTmpExtractPath.FullName, overwriteFiles: true);
+            } else
+            {
+                await using var uncompressedStream = new GZipStream(zStream, CompressionMode.Decompress);
+                await TarFile.ExtractToDirectoryAsync(uncompressedStream, qdrantTmpExtractPath.FullName, true);   
+            }
+        }
+
+        //
+        // Copy the database to the target directory:
+        //
+        Console.Write(" deploying ...");
+        var database = GetDatabasePath(rid);
+        if (string.IsNullOrWhiteSpace(database.Path))
+        {
+            Console.WriteLine($" failed to find the database path for {rid.ToUserFriendlyName()}");
+            return;
+        }
+
+        var qdrantDBSourcePath = Path.Join(qdrantTmpExtractPath.FullName, database.Path);
+        var qdrantDBTargetPath = Path.Join(cwd, "resources", "databases", "qdrant",database.Filename);
+        if (!File.Exists(qdrantDBSourcePath))
+        {
+            Console.WriteLine($" failed to find the database file '{qdrantDBSourcePath}'");
+            return;
+        }
+
+        Directory.CreateDirectory(Path.Join(cwd, "resources", "databases", "qdrant"));
+        if (File.Exists(qdrantDBTargetPath))
+            File.Delete(qdrantDBTargetPath);
+
+        File.Copy(qdrantDBSourcePath, qdrantDBTargetPath);
+
+        //
+        // Cleanup:
+        //
+        Console.Write(" cleaning up ...");
+        File.Delete(qdrantTmpDownloadPath);
+        Directory.Delete(qdrantTmpExtractPath.FullName, true);
+
+        Console.WriteLine(" done.");
+    }
+
+    private static Database GetDatabasePath(RID rid) => rid switch
+    {
+        RID.OSX_ARM64 => new("qdrant", "qdrant-aarch64-apple-darwin"),
+        RID.OSX_X64 => new("qdrant", "qdrant-x86_64-apple-darwin"),
+
+        RID.LINUX_ARM64 => new("qdrant", "qdrant-aarch64-unknown-linux-musl"),
+        RID.LINUX_X64 => new("qdrant", "qdrant-x86_64-unknown-linux-gnu"),
+
+        RID.WIN_X64 => new("qdrant.exe", "qdrant-x86_64-pc-windows-msvc.exe"),
+        RID.WIN_ARM64 => new("qdrant.exe", "qdrant-aarch64-pc-windows-msvc.exe"),
+
+        _ => new(string.Empty, string.Empty),
+    };
+
+    private static string GetQdrantDownloadUrl(RID rid, string version)
+    {
+        var baseUrl = $"https://github.com/qdrant/qdrant/releases/download/v{version}/qdrant-";
+        return rid switch
+        {
+            RID.LINUX_ARM64 => $"{baseUrl}aarch64-unknown-linux-musl.tar.gz",
+            RID.LINUX_X64 => $"{baseUrl}x86_64-unknown-linux-gnu.tar.gz",
+
+            RID.OSX_ARM64 => $"{baseUrl}aarch64-apple-darwin.tar.gz",
+            RID.OSX_X64 => $"{baseUrl}x86_64-apple-darwin.tar.gz",
+
+            RID.WIN_X64 => $"{baseUrl}x86_64-pc-windows-msvc.zip",
+            RID.WIN_ARM64 => $"{baseUrl}x86_64-pc-windows-msvc.zip",
+
+            _ => string.Empty,
+        };
+    }
+}

app/Build/Commands/UpdateMetadataCommands.cs

@@ -112,6 +112,9 @@ public async Task Build()
 
         var pdfiumVersion = await this.ReadPdfiumVersion();
         await Pdfium.InstallAsync(rid, pdfiumVersion);
+
+        var qdrantVersion = await this.ReadQdrantVersion();
+        await Qdrant.InstallAsync(rid, qdrantVersion);
 
         Console.Write($"- Start .NET build for {rid.ToUserFriendlyName()} ...");
         await this.ReadCommandOutput(pathApp, "dotnet", $"clean --configuration release --runtime {rid.AsMicrosoftRid()}");
@@ -324,6 +327,16 @@ private async Task<string> ReadPdfiumVersion()
         return shortVersion;
     }
 
+    private async Task<string> ReadQdrantVersion()
+    {
+        const int QDRANT_VERSION_INDEX = 11;
+        var pathMetadata = Environment.GetMetadataPath();
+        var lines = await File.ReadAllLinesAsync(pathMetadata, Encoding.UTF8);
+        var currentQdrantVersion = lines[QDRANT_VERSION_INDEX].Trim();
+
+        return currentQdrantVersion;
+    }
+
     private async Task UpdateArchitecture(RID rid)
     {
         const int ARCHITECTURE_INDEX = 9;

app/MindWork AI Studio/Assistants/I18N/allTexts.lua

@@ -4510,6 +4510,9 @@ UI_TEXT_CONTENT["AISTUDIO::PAGES::ABOUT::T1282228996"] = "AI Studio runs with an
 -- This library is used to read PDF files. This is necessary, e.g., for using PDFs as a data source for a chat.
 UI_TEXT_CONTENT["AISTUDIO::PAGES::ABOUT::T1388816916"] = "This library is used to read PDF files. This is necessary, e.g., for using PDFs as a data source for a chat."
 
+-- Database version
+UI_TEXT_CONTENT["AISTUDIO::PAGES::ABOUT::T1420062548"] = "Database version"
+
 -- This library is used to extend the MudBlazor library. It provides additional components that are not part of the MudBlazor library.
 UI_TEXT_CONTENT["AISTUDIO::PAGES::ABOUT::T1421513382"] = "This library is used to extend the MudBlazor library. It provides additional components that are not part of the MudBlazor library."
 
@@ -4549,6 +4552,9 @@ UI_TEXT_CONTENT["AISTUDIO::PAGES::ABOUT::T1924365263"] = "This library is used t
 -- We use Rocket to implement the runtime API. This is necessary because the runtime must be able to communicate with the user interface (IPC). Rocket is a great framework for implementing web APIs in Rust.
 UI_TEXT_CONTENT["AISTUDIO::PAGES::ABOUT::T1943216839"] = "We use Rocket to implement the runtime API. This is necessary because the runtime must be able to communicate with the user interface (IPC). Rocket is a great framework for implementing web APIs in Rust."
 
+-- Copies the following to the clipboard
+UI_TEXT_CONTENT["AISTUDIO::PAGES::ABOUT::T2029659664"] = "Copies the following to the clipboard"
+
 -- Copies the server URL to the clipboard
 UI_TEXT_CONTENT["AISTUDIO::PAGES::ABOUT::T2037899437"] = "Copies the server URL to the clipboard"
 
@@ -4723,6 +4729,9 @@ UI_TEXT_CONTENT["AISTUDIO::PAGES::ABOUT::T855925638"] = "We use this library to
 -- For some data transfers, we need to encode the data in base64. This Rust library is great for this purpose.
 UI_TEXT_CONTENT["AISTUDIO::PAGES::ABOUT::T870640199"] = "For some data transfers, we need to encode the data in base64. This Rust library is great for this purpose."
 
+-- Qdrant is a vector similarity search engine and vector database. It provides a production-ready service with a convenient API to store, search, and manage points—vectors with an additional payload Qdrant is tailored to extended filtering support.
+UI_TEXT_CONTENT["AISTUDIO::PAGES::ABOUT::T95576615"] = "Qdrant is a vector similarity search engine and vector database. It provides a production-ready service with a convenient API to store, search, and manage points—vectors with an additional payload Qdrant is tailored to extended filtering support."
+
 -- Install Pandoc
 UI_TEXT_CONTENT["AISTUDIO::PAGES::ABOUT::T986578435"] = "Install Pandoc"