Merge pull request #652 from iceljc/features/add-knowledge-docs

Features/add knowledge docs

Author: iceljc

src/Infrastructure/BotSharp.Abstraction/Files/Models/KnowledgeFileModel.cs

@@ -5,6 +5,7 @@ public class KnowledgeFileModel
     public Guid FileId { get; set; }
     public string FileName { get; set; }
     public string FileExtension { get; set; }
+    public string FileSource { get; set; }
     public string ContentType { get; set; }
     public string FileUrl { get; set; }
     public DocMetaRefData? RefData { get; set; }

src/Infrastructure/BotSharp.Abstraction/Knowledges/Enums/KnowledgePayloadName.cs

@@ -11,4 +11,5 @@ public static class KnowledgePayloadName
     public static string FileId = "fileId";
     public static string FileName = "fileName";
     public static string FileSource = "fileSource";
+    public static string FileUrl = "fileUrl";
 }

src/Plugins/BotSharp.Plugin.KnowledgeBase/Helpers/TextChopper.cs renamed to src/Infrastructure/BotSharp.Abstraction/Knowledges/Helpers/TextChopper.cs

@@ -1,6 +1,6 @@
 using System.Text.RegularExpressions;
 
-namespace BotSharp.Plugin.KnowledgeBase.Helpers;
+namespace BotSharp.Abstraction.Knowledges.Helpers;
 
 public static class TextChopper
 {
@@ -14,18 +14,22 @@ public static List<string> Chop(string content, ChunkOption option)
     private static List<string> ChopByWord(string content, ChunkOption option)
     {
         var chunks = new List<string>();
-        var words = content.Split(' ').Where(x => !string.IsNullOrWhiteSpace(x)).ToList();
+        var words = content.Split(' ', StringSplitOptions.RemoveEmptyEntries).Where(x => !string.IsNullOrWhiteSpace(x)).ToList();
 
         var chunk = string.Empty;
         for (int i = 0; i < words.Count; i++)
         {
-            chunk += words[i] + " ";
+            chunk += words[i];
             if (chunk.Length > option.Size)
             {
                 chunks.Add(chunk.Trim());
                 chunk = string.Empty;
                 i -= option.Conjunction;
             }
+            else
+            {
+                chunk += " ";
+            }
         }
 
         if (chunks.IsNullOrEmpty() && !string.IsNullOrEmpty(chunk))

src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeService.cs

@@ -22,7 +22,8 @@ public interface IKnowledgeService
     #endregion
 
     #region Document
-    Task<UploadKnowledgeResponse> UploadKnowledgeDocuments(string collectionName, IEnumerable<ExternalFileModel> files);
+    Task<UploadKnowledgeResponse> UploadDocumentsToKnowledge(string collectionName, IEnumerable<ExternalFileModel> files);
+    Task<bool> ImportDocumentContentToKnowledge(string collectionName, string fileName, string fileSource, IEnumerable<string> contents, DocMetaRefData? refData = null);
     Task<bool> DeleteKnowledgeDocument(string collectionName, Guid fileId);
     Task<PagedItems<KnowledgeFileModel>> GetPagedKnowledgeDocuments(string collectionName, KnowledgeFileFilter filter);
     Task<FileBinaryDataModel> GetKnowledgeDocumentBinaryData(string collectionName, Guid fileId);

src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeDocMetaData.cs

@@ -42,6 +42,9 @@ public class DocMetaRefData
     [JsonPropertyName("name")]
     public string Name { get; set; }
 
+    [JsonPropertyName("type")]
+    public string Type { get; set; }
+
     [JsonPropertyName("url")]
     public string Url { get; set; }
 

src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeFileFilter.cs

@@ -3,4 +3,6 @@ namespace BotSharp.Abstraction.Knowledges.Models;
 public class KnowledgeFileFilter : Pagination
 {
     public IEnumerable<Guid>? FileIds { get; set; }
+
+    public IEnumerable<string>? FileSources { get; set; }
 }

src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.KnowledgeBase.cs

@@ -182,10 +182,19 @@ public PagedItems<KnowledgeDocMetaData> GetKnowledgeBaseFileMeta(string collecti
             var matched = true;
 
             // Apply filter
-            if (filter != null && !filter.FileIds.IsNullOrEmpty())
+            if (filter != null)
             {
-                matched = matched && filter.FileIds.Contains(metaData.FileId);
+                if (!filter.FileIds.IsNullOrEmpty())
+                {
+                    matched = matched && filter.FileIds.Contains(metaData.FileId);
+                }
+
+                if (!filter.FileSources.IsNullOrEmpty())
+                {
+                    matched = matched & filter.FileSources.Contains(metaData.FileSource);
+                }
             }
+            
 
             if (!matched) continue;
 

src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBaseController.cs

@@ -115,7 +115,7 @@ public async Task<bool> DeleteVectorCollectionAllData([FromRoute] string collect
     [HttpPost("/knowledge/document/{collection}/upload")]
     public async Task<UploadKnowledgeResponse> UploadKnowledgeDocuments([FromRoute] string collection, [FromBody] VectorKnowledgeUploadRequest request)
     {
-        var response = await _knowledgeService.UploadKnowledgeDocuments(collection, request.Files);
+        var response = await _knowledgeService.UploadDocumentsToKnowledge(collection, request.Files);
         return response;
     }
 
@@ -138,7 +138,7 @@ public async Task<UploadKnowledgeResponse> UploadKnowledgeDocuments([FromRoute]
             });
         }
 
-        var response = await _knowledgeService.UploadKnowledgeDocuments(collection, docs);
+        var response = await _knowledgeService.UploadDocumentsToKnowledge(collection, docs);
         return response;
     }
 
@@ -149,7 +149,7 @@ public async Task<bool> DeleteKnowledgeDocument([FromRoute] string collection, [
         return response;
     }
 
-    [HttpPost("/knowledge/document/{collection}/list")]
+    [HttpPost("/knowledge/document/{collection}/page")]
     public async Task<PagedItems<KnowledgeFileViewModel>> GetPagedKnowledgeDocuments([FromRoute] string collection, [FromBody] GetKnowledgeDocsRequest request)
     {
         var data = await _knowledgeService.GetPagedKnowledgeDocuments(collection, new KnowledgeFileFilter

src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/KnowledgeFileViewModel.cs

@@ -10,6 +10,9 @@ public class KnowledgeFileViewModel
     [JsonPropertyName("file_name")]
     public string FileName { get; set; }
 
+    [JsonPropertyName("file_source")]
+    public string FileSource { get; set; }
+
     [JsonPropertyName("file_extension")]
     public string FileExtension { get; set; }
 
@@ -29,6 +32,7 @@ public static KnowledgeFileViewModel From(KnowledgeFileModel model)
         {
             FileId = model.FileId,
             FileName = model.FileName,
+            FileSource = model.FileSource,
             FileExtension = model.FileExtension,
             ContentType = model.ContentType,
             FileUrl = model.FileUrl,

src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Document.cs

@@ -1,6 +1,7 @@
 using BotSharp.Abstraction.Files;
 using BotSharp.Abstraction.Files.Models;
 using BotSharp.Abstraction.Files.Utilities;
+using BotSharp.Abstraction.Knowledges.Helpers;
 using BotSharp.Abstraction.VectorStorage.Enums;
 using System.Net.Http;
 using System.Net.Mime;
@@ -9,7 +10,7 @@ namespace BotSharp.Plugin.KnowledgeBase.Services;
 
 public partial class KnowledgeService
 {
-    public async Task<UploadKnowledgeResponse> UploadKnowledgeDocuments(string collectionName, IEnumerable<ExternalFileModel> files)
+    public async Task<UploadKnowledgeResponse> UploadDocumentsToKnowledge(string collectionName, IEnumerable<ExternalFileModel> files)
     {
         if (string.IsNullOrWhiteSpace(collectionName) || files.IsNullOrEmpty())
         {
@@ -89,6 +90,50 @@ public async Task<UploadKnowledgeResponse> UploadKnowledgeDocuments(string colle
     }
 
 
+    public async Task<bool> ImportDocumentContentToKnowledge(string collectionName, string fileName, string fileSource,
+        IEnumerable<string> contents, DocMetaRefData? refData = null)
+    {
+        if (string.IsNullOrWhiteSpace(collectionName)
+            || string.IsNullOrWhiteSpace(fileName)
+            || contents.IsNullOrEmpty())
+        {
+            return false;
+        }
+
+        try
+        {
+            var db = _services.GetRequiredService<IBotSharpRepository>();
+            var userId = await GetUserId();
+            var vectorStoreProvider = _settings.VectorDb.Provider;
+            var fileId = Guid.NewGuid();
+            var contentType = FileUtility.GetFileContentType(fileName);
+
+            var dataIds = await SaveToVectorDb(collectionName, fileId, fileName, contents, fileSource, fileUrl: refData?.Url);
+            db.SaveKnolwedgeBaseFileMeta(new KnowledgeDocMetaData
+            {
+                Collection = collectionName,
+                FileId = fileId,
+                FileName = fileName,
+                FileSource = fileSource,
+                ContentType = contentType,
+                VectorStoreProvider = vectorStoreProvider,
+                VectorDataIds = dataIds,
+                RefData = refData,
+                CreateDate = DateTime.UtcNow,
+                CreateUserId = userId
+            });
+            return true;
+        }
+        catch (Exception ex)
+        {
+            _logger.LogWarning($"Error when importing doc content to knowledgebase ({collectionName}-{fileName})" +
+                $"\r\n{ex.Message}" +
+                $"\r\n{ex.InnerException}");
+            return false;
+        }
+    }
+
+
     public async Task<bool> DeleteKnowledgeDocument(string collectionName, Guid fileId)
     {
         if (string.IsNullOrWhiteSpace(collectionName))
@@ -154,6 +199,7 @@ public async Task<PagedItems<KnowledgeFileModel>> GetPagedKnowledgeDocuments(str
         {
             FileId = x.FileId,
             FileName = x.FileName,
+            FileSource = x.FileSource,
             FileExtension = Path.GetExtension(x.FileName),
             ContentType = x.ContentType,
             FileUrl = fileStorage.GetKnowledgeBaseFileUrl(collectionName, vectorStoreProvider, x.FileId, x.FileName),
@@ -259,7 +305,7 @@ private async Task<IEnumerable<string>> ReadTxt(byte[] bytes)
         var lines = TextChopper.Chop(content, new ChunkOption
         {
             Size = 1024,
-            Conjunction = 32,
+            Conjunction = 12,
             SplitByWord = true,
         });
         return lines;
@@ -282,7 +328,7 @@ private bool SaveDocument(string collectionName, string vectorStoreProvider, Gui
 
     private async Task<IEnumerable<string>> SaveToVectorDb(
         string collectionName, Guid fileId, string fileName, IEnumerable<string> contents,
-        string fileSource = KnowledgeDocSource.Api, string vectorDataSource = VectorDataSource.File)
+        string fileSource = KnowledgeDocSource.Api, string vectorDataSource = VectorDataSource.File, string? fileUrl = null)
     {
         if (contents.IsNullOrEmpty())
         {
@@ -293,19 +339,25 @@ private async Task<IEnumerable<string>> SaveToVectorDb(
         var vectorDb = GetVectorDb();
         var textEmbedding = GetTextEmbedding(collectionName);
 
+        var payload = new Dictionary<string, string>
+        {
+            { KnowledgePayloadName.DataSource, vectorDataSource },
+            { KnowledgePayloadName.FileId, fileId.ToString() },
+            { KnowledgePayloadName.FileName, fileName },
+            { KnowledgePayloadName.FileSource, fileSource }
+        };
+
+        if (!string.IsNullOrWhiteSpace(fileUrl))
+        {
+            payload[KnowledgePayloadName.FileUrl] = fileUrl;
+        }
+
         for (int i = 0; i < contents.Count(); i++)
         {
             var content = contents.ElementAt(i);
             var vector = await textEmbedding.GetVectorAsync(content);
             var dataId = Guid.NewGuid();
-            var saved = await vectorDb.Upsert(collectionName, dataId, vector, content, new Dictionary<string, string>
-            {
-                { KnowledgePayloadName.DataSource, vectorDataSource },
-                { KnowledgePayloadName.FileId, fileId.ToString() },
-                { KnowledgePayloadName.FileName, fileName },
-                { KnowledgePayloadName.FileSource, fileSource },
-                { "textNumber", $"{i + 1}" }
-            });
+            var saved = await vectorDb.Upsert(collectionName, dataId, vector, content, payload);
 
             if (!saved) continue;