add vector storage snapshots

Author: Jicheng Lu

src/Infrastructure/BotSharp.Abstraction/Files/Utilities/FileUtility.cs

@@ -47,6 +47,17 @@ public static string BuildFileDataFromFile(IFormFile file)
         return $"data:{contentType};base64,{base64}";
     }
 
+    public static BinaryData BuildBinaryDataFromFile(IFormFile file)
+    {
+        using var stream = new MemoryStream();
+        file.CopyTo(stream);
+        stream.Position = 0;
+        var binary = BinaryData.FromStream(stream);
+        stream.Close();
+
+        return binary;
+    }
+
     public static string GetFileContentType(string fileName)
     {
         string contentType;

src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeService.cs

@@ -60,6 +60,14 @@ Task<bool> ImportDocumentContentToKnowledge(string collectionName, string fileNa
     Task<FileBinaryDataModel> GetKnowledgeDocumentBinaryData(string collectionName, Guid fileId);
     #endregion
 
+    #region Snapshot
+    Task<IEnumerable<VectorCollectionSnapshot>> GetVectorCollectionSnapshots(string collectionName);
+    Task<VectorCollectionSnapshot?> CreateVectorCollectionSnapshot(string collectionName);
+    Task<BinaryData> DownloadVectorCollectionSnapshot(string collectionName, string snapshotFileName);
+    Task<bool> RecoverVectorCollectionFromSnapshot(string collectionName, string snapshotFileName, BinaryData snapshotData);
+    Task<bool> DeleteVectorCollectionSnapshot(string collectionName, string snapshotName);
+    #endregion
+
     #region Common
     Task<bool> RefreshVectorKnowledgeConfigs(VectorCollectionConfigsModel configs);
     #endregion

src/Infrastructure/BotSharp.Abstraction/VectorStorage/IVectorDb.cs

@@ -6,14 +6,36 @@ public interface IVectorDb
 {
     string Provider { get; }
 
-    Task<bool> DoesCollectionExist(string collectionName);
-    Task<IEnumerable<string>> GetCollections();
-    Task<StringIdPagedItems<VectorCollectionData>> GetPagedCollectionData(string collectionName, VectorFilter filter);
-    Task<IEnumerable<VectorCollectionData>> GetCollectionData(string collectionName, IEnumerable<Guid> ids, bool withPayload = false, bool withVector = false);
-    Task<bool> CreateCollection(string collectionName, int dimension);
-    Task<bool> DeleteCollection(string collectionName);
-    Task<bool> Upsert(string collectionName, Guid id, float[] vector, string text, Dictionary<string, object>? payload = null);
-    Task<IEnumerable<VectorCollectionData>> Search(string collectionName, float[] vector, IEnumerable<string>? fields, int limit = 5, float confidence = 0.5f, bool withVector = false);
-    Task<bool> DeleteCollectionData(string collectionName, List<Guid> ids);
-    Task<bool> DeleteCollectionAllData(string collectionName);
+    Task<bool> DoesCollectionExist(string collectionName)
+        => throw new NotImplementedException();
+    Task<IEnumerable<string>> GetCollections()
+        => throw new NotImplementedException();
+    Task<StringIdPagedItems<VectorCollectionData>> GetPagedCollectionData(string collectionName, VectorFilter filter)
+        => throw new NotImplementedException();
+    Task<IEnumerable<VectorCollectionData>> GetCollectionData(string collectionName, IEnumerable<Guid> ids,
+        bool withPayload = false, bool withVector = false)
+        => throw new NotImplementedException();
+    Task<bool> CreateCollection(string collectionName, int dimension)
+        => throw new NotImplementedException();
+    Task<bool> DeleteCollection(string collectionName)
+        => throw new NotImplementedException();
+    Task<bool> Upsert(string collectionName, Guid id, float[] vector, string text, Dictionary<string, object>? payload = null)
+        => throw new NotImplementedException();
+    Task<IEnumerable<VectorCollectionData>> Search(string collectionName, float[] vector, IEnumerable<string>? fields,
+        int limit = 5, float confidence = 0.5f, bool withVector = false)
+        => throw new NotImplementedException();
+    Task<bool> DeleteCollectionData(string collectionName, List<Guid> ids)
+        => throw new NotImplementedException();
+    Task<bool> DeleteCollectionAllData(string collectionName)
+        => throw new NotImplementedException();
+    Task<IEnumerable<VectorCollectionSnapshot>> GetCollectionSnapshots(string collectionName)
+        => throw new NotImplementedException();
+    Task<VectorCollectionSnapshot?> CreateCollectionShapshot(string collectionName)
+        => throw new NotImplementedException();
+    Task<BinaryData> DownloadCollectionSnapshot(string collectionName, string snapshotFileName)
+        => throw new NotImplementedException();
+    Task<bool> RecoverCollectionFromShapshot(string collectionName, string snapshotFileName, BinaryData snapshotData)
+        => throw new NotImplementedException();
+    Task<bool> DeleteCollectionShapshot(string collectionName, string snapshotName)
+        => throw new NotImplementedException();
 }

src/Infrastructure/BotSharp.Abstraction/VectorStorage/Models/Snapshot/VectorCollectionSnapshot.cs

@@ -0,0 +1,9 @@
+namespace BotSharp.Abstraction.VectorStorage.Models;
+
+public class VectorCollectionSnapshot
+{
+    public string Name { get; set; } = default!;
+    public long Size { get; set; }
+    public DateTime CreatedTime { get; set; }
+    public string? CheckSum { get; set; }
+}

src/Infrastructure/BotSharp.OpenAPI/Controllers/DashboardController.cs

@@ -19,16 +19,16 @@ public DashboardController(IServiceProvider services,
     }
     #region User Components
     [HttpGet("/dashboard/components")]
-    public async Task<UserDashboardModel> GetComponents()
+    public async Task<UserDashboardViewModel> GetComponents()
     {
         var userService = _services.GetRequiredService<IUserService>();
         var dashboardProfile = await userService.GetDashboard();
-        if (dashboardProfile == null) return new UserDashboardModel();
+        if (dashboardProfile == null) return new();
 
-        var result = new UserDashboardModel
+        var result = new UserDashboardViewModel
         {
             ConversationList = dashboardProfile.ConversationList.Select(
-                x => new UserDashboardConversationModel
+                x => new UserDashboardConversationViewModel
                 {
                     Name = x.Name,
                     ConversationId = x.ConversationId,
@@ -40,7 +40,7 @@ public async Task<UserDashboardModel> GetComponents()
     }
 
     [HttpPost("/dashboard/component/conversation")]
-    public async Task UpdateDashboardConversationInstruction(UserDashboardConversationModel dashConv)
+    public async Task UpdateDashboardConversationInstruction(UserDashboardConversationViewModel dashConv)
     {
         if (string.IsNullOrEmpty(dashConv.Name) && string.IsNullOrEmpty(dashConv.Instruction)) 
         {

src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBaseController.cs

@@ -12,7 +12,9 @@ public class KnowledgeBaseController : ControllerBase
     private readonly IKnowledgeService _knowledgeService;
     private readonly IServiceProvider _services;
 
-    public KnowledgeBaseController(IKnowledgeService knowledgeService, IServiceProvider services)
+    public KnowledgeBaseController(
+        IKnowledgeService knowledgeService,
+        IServiceProvider services)
     {
         _knowledgeService = knowledgeService;
         _services = services;
@@ -117,6 +119,46 @@ public async Task<bool> DeleteVectorCollectionAllData([FromRoute] string collect
     #endregion
 
 
+    #region Snapshot
+    [HttpGet("/knowledge/vector/{collection}/snapshots")]
+    public async Task<IEnumerable<VectorCollectionSnapshotViewModel>> GetVectorCollectionSnapshots([FromRoute] string collection)
+    {
+        var snapshots = await _knowledgeService.GetVectorCollectionSnapshots(collection);
+        return snapshots.Select(x => VectorCollectionSnapshotViewModel.From(x));
+    }
+
+    [HttpPost("/knowledge/vector/{collection}/snapshot")]
+    public async Task<VectorCollectionSnapshotViewModel?> CreateVectorCollectionSnapshot([FromRoute] string collection)
+    {
+        var snapshot = await _knowledgeService.CreateVectorCollectionSnapshot(collection);
+        return VectorCollectionSnapshotViewModel.From(snapshot);
+    }
+
+    [HttpGet("/knowledge/vector/{collection}/snapshot")]
+    public async Task<IActionResult> GetVectorCollectionSnapshot([FromRoute] string collection, [FromQuery] string snapshotFileName)
+    {
+        var snapshot = await _knowledgeService.DownloadVectorCollectionSnapshot(collection, snapshotFileName);
+        return BuildFileResult(snapshotFileName, snapshot);
+    }
+
+    [HttpPost("/knowledge/vector/{collection}/snapshot/recover")]
+    public async Task<bool> RecoverVectorCollectionFromSnapshot([FromRoute] string collection, IFormFile snapshotFile)
+    {
+        var fileName = snapshotFile.FileName;
+        var binary = FileUtility.BuildBinaryDataFromFile(snapshotFile);
+        var done = await _knowledgeService.RecoverVectorCollectionFromSnapshot(collection, fileName, binary);
+        return done;
+    }
+
+    [HttpDelete("/knowledge/vector/{collection}/snapshot")]
+    public async Task<bool> DeleteVectorCollectionSnapshots([FromRoute] string collection, [FromBody] DeleteVectorCollectionSnapshotRequest request)
+    {
+        var done = await _knowledgeService.DeleteVectorCollectionSnapshot(collection, request.SnapshotName);
+        return done;
+    }
+    #endregion
+
+
     #region Document
     [HttpPost("/knowledge/document/{collection}/upload")]
     public async Task<UploadKnowledgeResponse> UploadKnowledgeDocuments([FromRoute] string collection, [FromBody] VectorKnowledgeUploadRequest request)
@@ -187,7 +229,6 @@ public async Task<IActionResult> GetKnowledgeDocument([FromRoute] string collect
     #endregion
 
 
-
     #region Graph
     [HttpPost("/knowledge/graph/search")]
     public async Task<GraphKnowledgeViewModel> SearchGraphKnowledge([FromBody] SearchGraphKnowledgeRequest request)
@@ -214,4 +255,13 @@ public async Task<string> RefreshVectorCollectionConfigs([FromBody] VectorCollec
         return saved ? "Success" : "Fail";
     }
     #endregion
+
+    #region Private methods
+    private FileStreamResult BuildFileResult(string fileName, BinaryData fileData)
+    {
+        var stream = fileData.ToStream();
+        stream.Position = 0;
+        return File(stream, "application/octet-stream", Path.GetFileName(fileName));
+    }
+    #endregion
 }