Merge pull request #786 from iceljc/features/add-crontab-storage

iceljc · web-flow · commit eac490e8ed0f · 2024-12-08T18:37:42.000-06:00
add crontab storage
diff --git a/BotSharp.sln b/BotSharp.sln
@@ -121,6 +121,8 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "BotSharp.Core.SideCar", "sr
 EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "BotSharp.Plugin.VertexAI", "src\Plugins\BotSharp.Plugin.LangChain\BotSharp.Plugin.VertexAI.csproj", "{7DA2DCD0-551B-432E-AA5C-22DDD3ED459B}"
 EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "BotSharp.Core.Crontab", "src\Infrastructure\BotSharp.Core.Crontab\BotSharp.Core.Crontab.csproj", "{F812BAAE-5A7D-4DF7-8E71-70696B51C61F}"
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Any CPU = Debug|Any CPU
@@ -489,6 +491,14 @@ Global
 		{7DA2DCD0-551B-432E-AA5C-22DDD3ED459B}.Release|Any CPU.Build.0 = Release|Any CPU
 		{7DA2DCD0-551B-432E-AA5C-22DDD3ED459B}.Release|x64.ActiveCfg = Release|Any CPU
 		{7DA2DCD0-551B-432E-AA5C-22DDD3ED459B}.Release|x64.Build.0 = Release|Any CPU
+		{F812BAAE-5A7D-4DF7-8E71-70696B51C61F}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{F812BAAE-5A7D-4DF7-8E71-70696B51C61F}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{F812BAAE-5A7D-4DF7-8E71-70696B51C61F}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{F812BAAE-5A7D-4DF7-8E71-70696B51C61F}.Debug|x64.Build.0 = Debug|Any CPU
+		{F812BAAE-5A7D-4DF7-8E71-70696B51C61F}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{F812BAAE-5A7D-4DF7-8E71-70696B51C61F}.Release|Any CPU.Build.0 = Release|Any CPU
+		{F812BAAE-5A7D-4DF7-8E71-70696B51C61F}.Release|x64.ActiveCfg = Release|Any CPU
+		{F812BAAE-5A7D-4DF7-8E71-70696B51C61F}.Release|x64.Build.0 = Release|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE
@@ -547,6 +557,7 @@ Global
 		{F57F4862-F8D4-44A1-AC12-5C131B5C9785} = {51AFE054-AE99-497D-A593-69BAEFB5106F}
 		{6D3A54F9-4792-41DB-BE7D-4F7B1D918EAE} = {E29DC6C4-5E57-48C5-BCB0-6B8F84782749}
 		{7DA2DCD0-551B-432E-AA5C-22DDD3ED459B} = {D5293208-2BEF-42FC-A64C-5954F61720BA}
+		{F812BAAE-5A7D-4DF7-8E71-70696B51C61F} = {E29DC6C4-5E57-48C5-BCB0-6B8F84782749}
 	EndGlobalSection
 	GlobalSection(ExtensibilityGlobals) = postSolution
 		SolutionGuid = {A9969D89-C98B-40A5-A12B-FC87E55B3A19}
diff --git a/src/Infrastructure/BotSharp.Abstraction/BotSharp.Abstraction.csproj b/src/Infrastructure/BotSharp.Abstraction/BotSharp.Abstraction.csproj
@@ -1,4 +1,4 @@
-<Project Sdk="Microsoft.NET.Sdk">
+﻿<Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
     <TargetFramework>$(TargetFramework)</TargetFramework>
diff --git a/src/Infrastructure/BotSharp.Abstraction/Crontab/Models/CrontabItem.cs b/src/Infrastructure/BotSharp.Abstraction/Crontab/Models/CrontabItem.cs
@@ -1,12 +1,25 @@
-namespace BotSharp.Core.Crontab.Models;
+namespace BotSharp.Abstraction.Crontab.Models;
 
 public class CrontabItem : ScheduleTaskArgs
 {
+    [JsonPropertyName("id")]
+    public string Id { get; set; }
+
+    [JsonPropertyName("user_id")]
     public string UserId { get; set; } = null!;
+
+    [JsonPropertyName("agent_id")]
     public string AgentId { get; set; } = null!;
+
+    [JsonPropertyName("conversation_id")]
     public string ConversationId { get; set; } = null!;
+
+    [JsonPropertyName("execution_result")]
     public string ExecutionResult { get; set; } = null!;
 
+    [JsonPropertyName("created_time")]
+    public DateTime CreatedTime { get; set; } = DateTime.UtcNow;
+
     public override string ToString()
     {
         return $"{Topic}: {Description} [AgentId: {AgentId}, UserId: {UserId}]";
diff --git a/src/Infrastructure/BotSharp.Abstraction/Crontab/Models/CrontabItemFilter.cs b/src/Infrastructure/BotSharp.Abstraction/Crontab/Models/CrontabItemFilter.cs
@@ -0,0 +1,26 @@
+namespace BotSharp.Abstraction.Crontab.Models;
+
+public class CrontabItemFilter : Pagination
+{
+    [JsonPropertyName("user_ids")]
+    public IEnumerable<string>? UserIds { get; set; }
+
+    [JsonPropertyName("agent_ids")]
+    public IEnumerable<string>? AgentIds { get; set; }
+
+    [JsonPropertyName("conversation_ids")]
+    public IEnumerable<string>? ConversationIds { get; set; }
+
+    [JsonPropertyName("topics")]
+    public IEnumerable<string>? Topics { get; set; }
+
+    public CrontabItemFilter()
+    {
+        
+    }
+
+    public static CrontabItemFilter Empty()
+    {
+        return new CrontabItemFilter();
+    }
+}
diff --git a/src/Infrastructure/BotSharp.Abstraction/Crontab/Models/ScheduleTaskArgs.cs b/src/Infrastructure/BotSharp.Abstraction/Crontab/Models/ScheduleTaskArgs.cs
@@ -1,6 +1,4 @@
-using System.Text.Json.Serialization;
-
-namespace BotSharp.Core.Crontab.Models;
+namespace BotSharp.Abstraction.Crontab.Models;
 
 public class ScheduleTaskArgs
 {
diff --git a/src/Infrastructure/BotSharp.Abstraction/Repositories/IBotSharpRepository.cs b/src/Infrastructure/BotSharp.Abstraction/Repositories/IBotSharpRepository.cs
@@ -145,4 +145,9 @@ public interface IBotSharpRepository : IHaveServiceProvider
     bool DeleteKnolwedgeBaseFileMeta(string collectionName, string vectorStoreProvider, Guid? fileId = null);
     PagedItems<KnowledgeDocMetaData> GetKnowledgeBaseFileMeta(string collectionName, string vectorStoreProvider, KnowledgeFileFilter filter);
     #endregion
+
+    #region Crontab
+    bool InsertCrontabItem(CrontabItem item) => throw new NotImplementedException();
+    PagedItems<CrontabItem> GetCrontabItems(CrontabItemFilter filter) => throw new NotImplementedException();
+    #endregion
 }
diff --git a/src/Infrastructure/BotSharp.Abstraction/Using.cs b/src/Infrastructure/BotSharp.Abstraction/Using.cs
@@ -17,4 +17,5 @@
 global using BotSharp.Abstraction.Messaging.Enums;
 global using BotSharp.Abstraction.Files.Models;
 global using BotSharp.Abstraction.Files.Enums;
-global using BotSharp.Abstraction.Knowledges.Models;
+global using BotSharp.Abstraction.Knowledges.Models;
+global using BotSharp.Abstraction.Crontab.Models;
diff --git a/src/Infrastructure/BotSharp.Core.Crontab/Abstraction/ICrontabHook.cs b/src/Infrastructure/BotSharp.Core.Crontab/Abstraction/ICrontabHook.cs
@@ -1,5 +1,3 @@
-using BotSharp.Core.Crontab.Models;
-
 namespace BotSharp.Core.Crontab.Abstraction;
 
 public interface ICrontabHook
diff --git a/src/Infrastructure/BotSharp.Core.Crontab/Abstraction/ICrontabService.cs b/src/Infrastructure/BotSharp.Core.Crontab/Abstraction/ICrontabService.cs
@@ -1,5 +1,3 @@
-using BotSharp.Core.Crontab.Models;
-
 namespace BotSharp.Core.Crontab.Abstraction;
 
 public interface ICrontabService
diff --git a/src/Infrastructure/BotSharp.Core.Crontab/BotSharp.Core.Crontab.csproj b/src/Infrastructure/BotSharp.Core.Crontab/BotSharp.Core.Crontab.csproj
@@ -21,12 +21,15 @@
   </ItemGroup>
 
   <ItemGroup>
-    <ProjectReference Include="..\BotSharp.Abstraction\BotSharp.Abstraction.csproj" />
     <ProjectReference Include="..\BotSharp.Core\BotSharp.Core.csproj" />
   </ItemGroup>
 
   <ItemGroup>
     <PackageReference Include="NCrontab" Version="3.3.3" />
   </ItemGroup>
 
+  <ItemGroup>
+    <Folder Include="Models\" />
+  </ItemGroup>
+
 </Project>
diff --git a/src/Infrastructure/BotSharp.Core.Crontab/Services/CrontabService.cs b/src/Infrastructure/BotSharp.Core.Crontab/Services/CrontabService.cs
@@ -17,7 +17,6 @@ limitations under the License.
 using BotSharp.Abstraction.Agents.Enums;
 using BotSharp.Abstraction.Conversations;
 using BotSharp.Abstraction.Routing;
-using BotSharp.Core.Crontab.Models;
 using BotSharp.Core.Infrastructures;
 
 
diff --git a/src/Infrastructure/BotSharp.Core.Crontab/Using.cs b/src/Infrastructure/BotSharp.Core.Crontab/Using.cs
@@ -4,10 +4,10 @@
 global using Microsoft.Extensions.DependencyInjection;
 
 global using BotSharp.Abstraction.Agents.Enums;
+global using BotSharp.Abstraction.Crontab.Models;
 global using BotSharp.Abstraction.Agents;
 global using BotSharp.Abstraction.Plugins;
 global using BotSharp.Abstraction.Conversations.Models;
 global using BotSharp.Abstraction.Functions;
 global using BotSharp.Core.Crontab.Services;
-global using BotSharp.Core.Crontab.Abstraction;
-global using BotSharp.Core.Crontab.Models;
+global using BotSharp.Core.Crontab.Abstraction;
diff --git a/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.Crontab.cs b/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.Crontab.cs
@@ -0,0 +1,96 @@
+using System.IO;
+
+namespace BotSharp.Core.Repository;
+
+public partial class FileRepository
+{
+    public bool InsertCrontabItem(CrontabItem item)
+    {
+        if (item == null)
+        {
+            return false;
+        }
+
+        try
+        {
+            var baseDir = Path.Combine(_dbSettings.FileRepository, CRONTAB_FOLDER);
+            item.Id = Guid.NewGuid().ToString();
+            var dir = Path.Combine(baseDir, item.Id);
+
+            if (Directory.Exists(dir))
+            {
+                return false;
+            }
+
+            Directory.CreateDirectory(dir);
+            Thread.Sleep(50);
+
+            var itemFile = Path.Combine(dir, CRONTAB_FILE);
+            var json = JsonSerializer.Serialize(item, _options);
+            File.WriteAllText(itemFile, json);
+            return true;
+        }
+        catch (Exception ex)
+        {
+            _logger.LogError($"Error when saving crontab item: {ex.Message}\r\n{ex.InnerException}");
+            return false;
+        }
+    }
+
+
+    public PagedItems<CrontabItem> GetCrontabItems(CrontabItemFilter filter)
+    {
+        
+        if (filter == null)
+        {
+            filter = CrontabItemFilter.Empty();
+        }
+
+        var records = new List<CrontabItem>();
+        var dir = Path.Combine(_dbSettings.FileRepository, CRONTAB_FOLDER);
+
+        if (!Directory.Exists(dir))
+        {
+            Directory.CreateDirectory(dir);
+        }
+
+        var totalDirs = Directory.GetDirectories(dir);
+        foreach (var d in totalDirs)
+        {
+            var file = Path.Combine(d, CRONTAB_FILE);
+            if (!File.Exists(file)) continue;
+
+            var json = File.ReadAllText(file);
+            var record = JsonSerializer.Deserialize<CrontabItem>(json, _options);
+            if (record == null) continue;
+
+            var matched = true;
+            if (filter?.AgentIds != null)
+            {
+                matched = matched && filter.AgentIds.Contains(record.AgentId);
+            }
+            if (filter?.ConversationIds != null)
+            {
+                matched = matched && filter.ConversationIds.Contains(record.ConversationId);
+            }
+            if (filter?.UserIds != null)
+            {
+                matched = matched && filter.UserIds.Contains(record.UserId);
+            }
+            if (filter?.Topics != null)
+            {
+                matched = matched && filter.Topics.Contains(record.Topic);
+            }
+
+            if (!matched) continue;
+
+            records.Add(record);
+        }
+
+        return new PagedItems<CrontabItem>
+        {
+            Items = records.OrderByDescending(x => x.CreatedTime).Skip(filter.Offset).Take(filter.Size),
+            Count = records.Count(),
+        };
+    }
+}
diff --git a/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.cs b/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.cs
@@ -55,6 +55,9 @@ public partial class FileRepository : IBotSharpRepository
     private const string PLUGIN_CONFIG_FILE = "config.json";
     private const string STATS_FILE = "stats.json";
 
+    private const string CRONTAB_FOLDER = "crontabs";
+    private const string CRONTAB_FILE = "crontab.json";
+
     public FileRepository(
         IServiceProvider services,
         BotSharpDatabaseSettings dbSettings,
@@ -88,7 +91,6 @@ public FileRepository(
     private List<Agent> _agents = new List<Agent>();
     private List<RoleAgent> _roleAgents = new List<RoleAgent>();
     private List<UserAgent> _userAgents = new List<UserAgent>();
-    private List<Conversation> _conversations = new List<Conversation>();
     private PluginConfig? _pluginConfig = null;
 
     private IQueryable<Role> Roles
diff --git a/src/Infrastructure/BotSharp.Core/Using.cs b/src/Infrastructure/BotSharp.Core/Using.cs
@@ -14,7 +14,7 @@
 global using BotSharp.Abstraction.Plugins;
 global using BotSharp.Abstraction.Agents;
 global using BotSharp.Abstraction.Conversations;
-global using BotSharp.Abstraction.Knowledges;
+global using BotSharp.Abstraction.Crontab.Models;
 global using BotSharp.Abstraction.Users;
 global using BotSharp.Abstraction.Roles;
 global using BotSharp.Abstraction.Roles.Models;
diff --git a/src/Plugins/BotSharp.Plugin.ChatHub/Hooks/ChatHubCrontabHook.cs b/src/Plugins/BotSharp.Plugin.ChatHub/Hooks/ChatHubCrontabHook.cs
@@ -1,5 +1,5 @@
+using BotSharp.Abstraction.Crontab.Models;
 using BotSharp.Core.Crontab.Abstraction;
-using BotSharp.Core.Crontab.Models;
 using Microsoft.AspNetCore.SignalR;
 
 namespace BotSharp.Plugin.ChatHub.Hooks;
diff --git a/src/Plugins/BotSharp.Plugin.MongoStorage/Collections/CrontabItemDocument.cs b/src/Plugins/BotSharp.Plugin.MongoStorage/Collections/CrontabItemDocument.cs
@@ -0,0 +1,53 @@
+using BotSharp.Abstraction.Crontab.Models;
+
+namespace BotSharp.Plugin.MongoStorage.Collections;
+
+public class CrontabItemDocument : MongoBase
+{
+    public string UserId { get; set; }
+    public string AgentId { get; set; }
+    public string ConversationId { get; set; }
+    public string ExecutionResult { get; set; }
+    public string Cron { get; set; }
+    public string Topic { get; set; }
+    public string Description { get; set; }
+    public string Script { get; set; }
+    public string Language { get; set; }
+    public DateTime CreatedTime { get; set; } = DateTime.UtcNow;
+
+    public static CrontabItem ToDomainModel(CrontabItemDocument item)
+    {
+        return new CrontabItem
+        {
+            Id = item.Id,
+            UserId = item.UserId,
+            AgentId = item.AgentId,
+            ConversationId = item.ConversationId,
+            ExecutionResult = item.ExecutionResult,
+            Cron = item.Cron,
+            Topic = item.Topic,
+            Description = item.Description,
+            Script = item.Script,
+            Language = item.Language,
+            CreatedTime = item.CreatedTime
+        };
+    }
+
+    public static CrontabItemDocument ToMongoModel(CrontabItem item)
+    {
+        return new CrontabItemDocument
+        {
+            Id = item.Id,
+            UserId = item.UserId,
+            AgentId = item.AgentId,
+            ConversationId = item.ConversationId,
+            ExecutionResult = item.ExecutionResult,
+            Cron = item.Cron,
+            Topic = item.Topic,
+            Description = item.Description,
+            Script = item.Script,
+            Language = item.Language,
+            CreatedTime = item.CreatedTime
+        };
+    }
+}
diff --git a/src/Plugins/BotSharp.Plugin.MongoStorage/MongoDbContext.cs b/src/Plugins/BotSharp.Plugin.MongoStorage/MongoDbContext.cs
@@ -165,4 +165,7 @@ public IMongoCollection<RoleDocument> Roles
 
     public IMongoCollection<RoleAgentDocument> RoleAgents
         => Database.GetCollection<RoleAgentDocument>($"{_collectionPrefix}_RoleAgents");
+
+    public IMongoCollection<CrontabItemDocument> CrontabItems
+        => Database.GetCollection<CrontabItemDocument>($"{_collectionPrefix}_CronTabItems");
 }
diff --git a/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Crontab.cs b/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Crontab.cs
diff --git a/src/Plugins/BotSharp.Plugin.SqlDriver/Hooks/SqlDriverCrontabHook.cs b/src/Plugins/BotSharp.Plugin.SqlDriver/Hooks/SqlDriverCrontabHook.cs
diff --git a/src/WebStarter/WebStarter.csproj b/src/WebStarter/WebStarter.csproj
diff --git a/src/WebStarter/appsettings.json b/src/WebStarter/appsettings.json

Original file line number	Diff line number	Diff line change
`@@ -1,4 +1,4 @@`
`1`		`-<Project Sdk="Microsoft.NET.Sdk">`
	`1`	`+<Project Sdk="Microsoft.NET.Sdk">`
`2`	`2`
`3`	`3`	`<PropertyGroup>`
`4`	`4`	`<TargetFramework>$(TargetFramework)</TargetFramework>`
Original file line number	Diff line number	Diff line change
`@@ -1,6 +1,4 @@`
`1`		`-using System.Text.Json.Serialization;`
`2`		`-`
`3`		`-namespace BotSharp.Core.Crontab.Models;`
	`1`	`+namespace BotSharp.Abstraction.Crontab.Models;`
`4`	`2`
`5`	`3`	`public class ScheduleTaskArgs`
`6`	`4`	`{`
Original file line number	Diff line number	Diff line change
`@@ -1,5 +1,3 @@`
`1`		`-using BotSharp.Core.Crontab.Models;`
`2`		`-`
`3`	`1`	`namespace BotSharp.Core.Crontab.Abstraction;`
`4`	`2`
`5`	`3`	`public interface ICrontabHook`