Merge pull request #653 from iceljc/master

refine knowledge doc filter

Author: iceljc

src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeService.cs

@@ -22,9 +22,37 @@ public interface IKnowledgeService
     #endregion
 
     #region Document
+    /// <summary>
+    /// Save documents and their contents to knowledgebase
+    /// </summary>
+    /// <param name="collectionName"></param>
+    /// <param name="files"></param>
+    /// <returns></returns>
     Task<UploadKnowledgeResponse> UploadDocumentsToKnowledge(string collectionName, IEnumerable<ExternalFileModel> files);
+    /// <summary>
+    /// Save document content to knowledgebase without saving the document
+    /// </summary>
+    /// <param name="collectionName"></param>
+    /// <param name="fileName"></param>
+    /// <param name="fileSource"></param>
+    /// <param name="contents"></param>
+    /// <param name="refData"></param>
+    /// <returns></returns>
     Task<bool> ImportDocumentContentToKnowledge(string collectionName, string fileName, string fileSource, IEnumerable<string> contents, DocMetaRefData? refData = null);
+    /// <summary>
+    /// Delete one document and its related knowledge in the collection
+    /// </summary>
+    /// <param name="collectionName"></param>
+    /// <param name="fileId"></param>
+    /// <returns></returns>
     Task<bool> DeleteKnowledgeDocument(string collectionName, Guid fileId);
+    /// <summary>
+    /// Delete all documents and their related knowledge in the collection
+    /// </summary>
+    /// <param name="collectionName"></param>
+    /// <param name="filter"></param>
+    /// <returns></returns>
+    Task<bool> DeleteKnowledgeDocuments(string collectionName, KnowledgeFileFilter filter);
     Task<PagedItems<KnowledgeFileModel>> GetPagedKnowledgeDocuments(string collectionName, KnowledgeFileFilter filter);
     Task<FileBinaryDataModel> GetKnowledgeDocumentBinaryData(string collectionName, Guid fileId);
     #endregion

src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeFileFilter.cs

@@ -4,5 +4,14 @@ public class KnowledgeFileFilter : Pagination
 {
     public IEnumerable<Guid>? FileIds { get; set; }
 
+    public IEnumerable<string>? FileNames { get; set; }
+
+    public IEnumerable<string>? ContentTypes { get; set; }
+
     public IEnumerable<string>? FileSources { get; set; }
+
+    public KnowledgeFileFilter()
+    {
+        
+    }
 }

src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.KnowledgeBase.cs

@@ -189,10 +189,20 @@ public PagedItems<KnowledgeDocMetaData> GetKnowledgeBaseFileMeta(string collecti
                     matched = matched && filter.FileIds.Contains(metaData.FileId);
                 }
 
+                if (!filter.FileNames.IsNullOrEmpty())
+                {
+                    matched = matched && filter.FileNames.Contains(metaData.FileName);
+                }
+
                 if (!filter.FileSources.IsNullOrEmpty())
                 {
                     matched = matched & filter.FileSources.Contains(metaData.FileSource);
                 }
+
+                if (!filter.ContentTypes.IsNullOrEmpty())
+                {
+                    matched = matched && filter.ContentTypes.Contains(metaData.ContentType);
+                }
             }
 
 

src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBaseController.cs

@@ -149,14 +149,17 @@ public async Task<bool> DeleteKnowledgeDocument([FromRoute] string collection, [
         return response;
     }
 
+    [HttpDelete("/knowledge/document/{collection}/delete")]
+    public async Task<bool> DeleteKnowledgeDocuments([FromRoute] string collection, [FromBody] GetKnowledgeDocsRequest request)
+    {
+        var response = await _knowledgeService.DeleteKnowledgeDocuments(collection, request);
+        return response;
+    }
+
     [HttpPost("/knowledge/document/{collection}/page")]
     public async Task<PagedItems<KnowledgeFileViewModel>> GetPagedKnowledgeDocuments([FromRoute] string collection, [FromBody] GetKnowledgeDocsRequest request)
     {
-        var data = await _knowledgeService.GetPagedKnowledgeDocuments(collection, new KnowledgeFileFilter
-        {
-            Page = request.Page,
-            Size = request.Size
-        });
+        var data = await _knowledgeService.GetPagedKnowledgeDocuments(collection, request);
 
         return new PagedItems<KnowledgeFileViewModel>
         {

src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Document.cs

@@ -3,6 +3,7 @@
 using BotSharp.Abstraction.Files.Utilities;
 using BotSharp.Abstraction.Knowledges.Helpers;
 using BotSharp.Abstraction.VectorStorage.Enums;
+using System.Collections;
 using System.Net.Http;
 using System.Net.Mime;
 
@@ -151,8 +152,8 @@ public async Task<bool> DeleteKnowledgeDocument(string collectionName, Guid file
             // Get doc meta data
             var pageData = db.GetKnowledgeBaseFileMeta(collectionName, vectorStoreProvider, new KnowledgeFileFilter
             {
-                FileIds = [ fileId ],
-                Size = 1
+                Size = 1,
+                FileIds = [ fileId ]
             });
 
             // Delete doc
@@ -177,6 +178,57 @@ public async Task<bool> DeleteKnowledgeDocument(string collectionName, Guid file
         }
     }
 
+    public async Task<bool> DeleteKnowledgeDocuments(string collectionName, KnowledgeFileFilter filter)
+    {
+        if (string.IsNullOrWhiteSpace(collectionName)) return false;
+
+        
+        var pageSize = filter.Size;
+        var innerFilter = new KnowledgeFileFilter
+        {
+            Page = 1,
+            Size = pageSize,
+            FileIds = filter.FileIds,
+            FileNames = filter.FileNames,
+            FileSources = filter.FileSources,
+            ContentTypes = filter.ContentTypes
+        };
+
+        var pageData = await GetPagedKnowledgeDocuments(collectionName, innerFilter);
+
+        var total = pageData.Count;
+        if (total == 0) return false;
+
+        var page = 1;
+        var totalPages = total % pageSize == 0 ? total / pageSize : total / pageSize + 1;
+
+        while (page <= totalPages)
+        {
+            if (page > 1)
+            {
+                pageData = await GetPagedKnowledgeDocuments(collectionName, innerFilter);
+            }
+
+            var fileIds = pageData.Items.Select(x => x.FileId).ToList();
+            foreach (var fileId in fileIds)
+            {
+                try
+                {
+                    await DeleteKnowledgeDocument(collectionName, fileId);
+                }
+                catch
+                {
+                    continue;
+                }
+            }
+
+            page++;
+        }
+
+        return true;
+    }
+
+
     public async Task<PagedItems<KnowledgeFileModel>> GetPagedKnowledgeDocuments(string collectionName, KnowledgeFileFilter filter)
     {
         if (string.IsNullOrWhiteSpace(collectionName))
@@ -189,11 +241,7 @@ public async Task<PagedItems<KnowledgeFileModel>> GetPagedKnowledgeDocuments(str
         var vectorStoreProvider = _settings.VectorDb.Provider;
 
         // Get doc meta data
-        var pagedData = db.GetKnowledgeBaseFileMeta(collectionName, vectorStoreProvider, new KnowledgeFileFilter
-        {
-            Page = filter.Page,
-            Size = filter.Size
-        });
+        var pagedData = db.GetKnowledgeBaseFileMeta(collectionName, vectorStoreProvider, filter);
 
         var files = pagedData.Items?.Select(x => new KnowledgeFileModel
         {
@@ -222,8 +270,8 @@ public async Task<FileBinaryDataModel> GetKnowledgeDocumentBinaryData(string col
         // Get doc binary data
         var pageData = db.GetKnowledgeBaseFileMeta(collectionName, vectorStoreProvider, new KnowledgeFileFilter
         {
-            FileIds = [ fileId ],
-            Size = 1
+            Size = 1,
+            FileIds = [ fileId ]
         });
 
         var metaData = pageData?.Items?.FirstOrDefault();
@@ -247,6 +295,7 @@ public async Task<FileBinaryDataModel> GetKnowledgeDocumentBinaryData(string col
     }
 
 
+
     #region Private methods
     /// <summary>
     /// Get file content type and file bytes

src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.KnowledgeBase.cs

@@ -191,10 +191,20 @@ public PagedItems<KnowledgeDocMetaData> GetKnowledgeBaseFileMeta(string collecti
                 docFilters.Add(builder.In(x => x.FileId, filter.FileIds));
             }
 
+            if (!filter.FileNames.IsNullOrEmpty())
+            {
+                docFilters.Add(builder.In(x => x.FileName, filter.FileNames));
+            }
+
             if (!filter.FileSources.IsNullOrEmpty())
             {
                 docFilters.Add(builder.In(x => x.FileSource, filter.FileSources));
             }
+
+            if (!filter.ContentTypes.IsNullOrEmpty())
+            {
+                docFilters.Add(builder.In(x => x.ContentType, filter.ContentTypes));
+            }
         }
 
         var filterDef = builder.And(docFilters);