Merge branch 'SciSharp:master' into master

Author: Oceania2018

src/Infrastructure/BotSharp.Abstraction/Files/Models/KnowledgeFileModel.cs

@@ -5,6 +5,7 @@ public class KnowledgeFileModel
     public Guid FileId { get; set; }
     public string FileName { get; set; }
     public string FileExtension { get; set; }
+    public string FileSource { get; set; }
     public string ContentType { get; set; }
     public string FileUrl { get; set; }
     public DocMetaRefData? RefData { get; set; }

src/Infrastructure/BotSharp.Abstraction/Knowledges/Enums/KnowledgePayloadName.cs

@@ -11,4 +11,5 @@ public static class KnowledgePayloadName
     public static string FileId = "fileId";
     public static string FileName = "fileName";
     public static string FileSource = "fileSource";
+    public static string FileUrl = "fileUrl";
 }

src/Plugins/BotSharp.Plugin.KnowledgeBase/Helpers/TextChopper.cs renamed to src/Infrastructure/BotSharp.Abstraction/Knowledges/Helpers/TextChopper.cs

@@ -1,6 +1,6 @@
 using System.Text.RegularExpressions;
 
-namespace BotSharp.Plugin.KnowledgeBase.Helpers;
+namespace BotSharp.Abstraction.Knowledges.Helpers;
 
 public static class TextChopper
 {
@@ -14,18 +14,22 @@ public static List<string> Chop(string content, ChunkOption option)
     private static List<string> ChopByWord(string content, ChunkOption option)
     {
         var chunks = new List<string>();
-        var words = content.Split(' ').Where(x => !string.IsNullOrWhiteSpace(x)).ToList();
+        var words = content.Split(' ', StringSplitOptions.RemoveEmptyEntries).Where(x => !string.IsNullOrWhiteSpace(x)).ToList();
 
         var chunk = string.Empty;
         for (int i = 0; i < words.Count; i++)
         {
-            chunk += words[i] + " ";
+            chunk += words[i];
             if (chunk.Length > option.Size)
             {
                 chunks.Add(chunk.Trim());
                 chunk = string.Empty;
                 i -= option.Conjunction;
             }
+            else
+            {
+                chunk += " ";
+            }
         }
 
         if (chunks.IsNullOrEmpty() && !string.IsNullOrEmpty(chunk))

src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeService.cs

@@ -22,8 +22,37 @@ public interface IKnowledgeService
     #endregion
 
     #region Document
-    Task<UploadKnowledgeResponse> UploadKnowledgeDocuments(string collectionName, IEnumerable<ExternalFileModel> files);
+    /// <summary>
+    /// Save documents and their contents to knowledgebase
+    /// </summary>
+    /// <param name="collectionName"></param>
+    /// <param name="files"></param>
+    /// <returns></returns>
+    Task<UploadKnowledgeResponse> UploadDocumentsToKnowledge(string collectionName, IEnumerable<ExternalFileModel> files);
+    /// <summary>
+    /// Save document content to knowledgebase without saving the document
+    /// </summary>
+    /// <param name="collectionName"></param>
+    /// <param name="fileName"></param>
+    /// <param name="fileSource"></param>
+    /// <param name="contents"></param>
+    /// <param name="refData"></param>
+    /// <returns></returns>
+    Task<bool> ImportDocumentContentToKnowledge(string collectionName, string fileName, string fileSource, IEnumerable<string> contents, DocMetaRefData? refData = null);
+    /// <summary>
+    /// Delete one document and its related knowledge in the collection
+    /// </summary>
+    /// <param name="collectionName"></param>
+    /// <param name="fileId"></param>
+    /// <returns></returns>
     Task<bool> DeleteKnowledgeDocument(string collectionName, Guid fileId);
+    /// <summary>
+    /// Delete all documents and their related knowledge in the collection
+    /// </summary>
+    /// <param name="collectionName"></param>
+    /// <param name="filter"></param>
+    /// <returns></returns>
+    Task<bool> DeleteKnowledgeDocuments(string collectionName, KnowledgeFileFilter filter);
     Task<PagedItems<KnowledgeFileModel>> GetPagedKnowledgeDocuments(string collectionName, KnowledgeFileFilter filter);
     Task<FileBinaryDataModel> GetKnowledgeDocumentBinaryData(string collectionName, Guid fileId);
     #endregion

src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeDocMetaData.cs

@@ -42,10 +42,13 @@ public class DocMetaRefData
     [JsonPropertyName("name")]
     public string Name { get; set; }
 
+    [JsonPropertyName("type")]
+    public string Type { get; set; }
+
     [JsonPropertyName("url")]
     public string Url { get; set; }
 
-    [JsonPropertyName("json_content")]
+    [JsonPropertyName("data")]
     [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
-    public string? JsonContent { get; set; }
+    public IDictionary<string, string>? Data { get; set; }
 }

src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeFileFilter.cs

@@ -3,4 +3,15 @@ namespace BotSharp.Abstraction.Knowledges.Models;
 public class KnowledgeFileFilter : Pagination
 {
     public IEnumerable<Guid>? FileIds { get; set; }
+
+    public IEnumerable<string>? FileNames { get; set; }
+
+    public IEnumerable<string>? ContentTypes { get; set; }
+
+    public IEnumerable<string>? FileSources { get; set; }
+
+    public KnowledgeFileFilter()
+    {
+        
+    }
 }

src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.KnowledgeBase.cs

@@ -182,10 +182,29 @@ public PagedItems<KnowledgeDocMetaData> GetKnowledgeBaseFileMeta(string collecti
             var matched = true;
 
             // Apply filter
-            if (filter != null && !filter.FileIds.IsNullOrEmpty())
+            if (filter != null)
             {
-                matched = matched && filter.FileIds.Contains(metaData.FileId);
+                if (!filter.FileIds.IsNullOrEmpty())
+                {
+                    matched = matched && filter.FileIds.Contains(metaData.FileId);
+                }
+
+                if (!filter.FileNames.IsNullOrEmpty())
+                {
+                    matched = matched && filter.FileNames.Contains(metaData.FileName);
+                }
+
+                if (!filter.FileSources.IsNullOrEmpty())
+                {
+                    matched = matched & filter.FileSources.Contains(metaData.FileSource);
+                }
+
+                if (!filter.ContentTypes.IsNullOrEmpty())
+                {
+                    matched = matched && filter.ContentTypes.Contains(metaData.ContentType);
+                }
             }
+            
 
             if (!matched) continue;
 

src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBaseController.cs

@@ -115,7 +115,7 @@ public async Task<bool> DeleteVectorCollectionAllData([FromRoute] string collect
     [HttpPost("/knowledge/document/{collection}/upload")]
     public async Task<UploadKnowledgeResponse> UploadKnowledgeDocuments([FromRoute] string collection, [FromBody] VectorKnowledgeUploadRequest request)
     {
-        var response = await _knowledgeService.UploadKnowledgeDocuments(collection, request.Files);
+        var response = await _knowledgeService.UploadDocumentsToKnowledge(collection, request.Files);
         return response;
     }
 
@@ -138,7 +138,7 @@ public async Task<UploadKnowledgeResponse> UploadKnowledgeDocuments([FromRoute]
             });
         }
 
-        var response = await _knowledgeService.UploadKnowledgeDocuments(collection, docs);
+        var response = await _knowledgeService.UploadDocumentsToKnowledge(collection, docs);
         return response;
     }
 
@@ -149,14 +149,17 @@ public async Task<bool> DeleteKnowledgeDocument([FromRoute] string collection, [
         return response;
     }
 
-    [HttpPost("/knowledge/document/{collection}/list")]
+    [HttpDelete("/knowledge/document/{collection}/delete")]
+    public async Task<bool> DeleteKnowledgeDocuments([FromRoute] string collection, [FromBody] GetKnowledgeDocsRequest request)
+    {
+        var response = await _knowledgeService.DeleteKnowledgeDocuments(collection, request);
+        return response;
+    }
+
+    [HttpPost("/knowledge/document/{collection}/page")]
     public async Task<PagedItems<KnowledgeFileViewModel>> GetPagedKnowledgeDocuments([FromRoute] string collection, [FromBody] GetKnowledgeDocsRequest request)
     {
-        var data = await _knowledgeService.GetPagedKnowledgeDocuments(collection, new KnowledgeFileFilter
-        {
-            Page = request.Page,
-            Size = request.Size
-        });
+        var data = await _knowledgeService.GetPagedKnowledgeDocuments(collection, request);
 
         return new PagedItems<KnowledgeFileViewModel>
         {

src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/KnowledgeFileViewModel.cs

@@ -10,6 +10,9 @@ public class KnowledgeFileViewModel
     [JsonPropertyName("file_name")]
     public string FileName { get; set; }
 
+    [JsonPropertyName("file_source")]
+    public string FileSource { get; set; }
+
     [JsonPropertyName("file_extension")]
     public string FileExtension { get; set; }
 
@@ -29,6 +32,7 @@ public static KnowledgeFileViewModel From(KnowledgeFileModel model)
         {
             FileId = model.FileId,
             FileName = model.FileName,
+            FileSource = model.FileSource,
             FileExtension = model.FileExtension,
             ContentType = model.ContentType,
             FileUrl = model.FileUrl,