Skip to content

Commit 3ea5b3a

Browse files
geffzhanggeffzhang
authored
Merge branch 'master' into upgrademcp
2 parents 38d1d4f + 11d315d commit 3ea5b3a

File tree

52 files changed

+1323
-303
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

52 files changed

+1323
-303
lines changed

‎BotSharp.sln‎

Lines changed: 44 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -133,6 +133,14 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "BotSharp.PizzaBot.MCPServer
133133
EndProject
134134
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "BotSharp.Core.MCP", "src\Infrastructure\BotSharp.Core.MCP\BotSharp.Core.MCP.csproj", "{534D1DD5-9D62-115C-F230-47080D76CE52}"
135135
EndProject
136+
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "BotSharp.Test.ComputerUse", "tests\BotSharp.Test.ComputerUse\BotSharp.Test.ComputerUse.csproj", "{C19D9AC1-97DD-8E65-E8DB-D295A095AA2D}"
137+
EndProject
138+
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "BotSharp.Plugin.OsDriver", "src\Plugins\BotSharp.Plugin.OsDriver\BotSharp.Plugin.OsDriver.csproj", "{B268E2F0-060F-8466-7D81-ABA4D735CA59}"
139+
EndProject
140+
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "BotSharp.Abstraction.ComputerUse", "src\Infrastructure\BotSharp.Abstraction.ComputerUse\BotSharp.Abstraction.ComputerUse.csproj", "{970BE341-9AC8-99A5-6572-E703C1E02FCB}"
141+
EndProject
142+
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "BotSharp.Test.BrowserUse", "tests\BotSharp.Test.BrowserUse\BotSharp.Test.BrowserUse.csproj", "{7D0DB012-9798-4BB9-B15B-A5B0B7B3B094}"
143+
EndProject
136144
Global
137145
GlobalSection(SolutionConfigurationPlatforms) = preSolution
138146
Debug|Any CPU = Debug|Any CPU
@@ -549,6 +557,38 @@ Global
549557
{534D1DD5-9D62-115C-F230-47080D76CE52}.Release|Any CPU.Build.0 = Release|Any CPU
550558
{534D1DD5-9D62-115C-F230-47080D76CE52}.Release|x64.ActiveCfg = Release|Any CPU
551559
{534D1DD5-9D62-115C-F230-47080D76CE52}.Release|x64.Build.0 = Release|Any CPU
560+
{C19D9AC1-97DD-8E65-E8DB-D295A095AA2D}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
561+
{C19D9AC1-97DD-8E65-E8DB-D295A095AA2D}.Debug|Any CPU.Build.0 = Debug|Any CPU
562+
{C19D9AC1-97DD-8E65-E8DB-D295A095AA2D}.Debug|x64.ActiveCfg = Debug|Any CPU
563+
{C19D9AC1-97DD-8E65-E8DB-D295A095AA2D}.Debug|x64.Build.0 = Debug|Any CPU
564+
{C19D9AC1-97DD-8E65-E8DB-D295A095AA2D}.Release|Any CPU.ActiveCfg = Release|Any CPU
565+
{C19D9AC1-97DD-8E65-E8DB-D295A095AA2D}.Release|Any CPU.Build.0 = Release|Any CPU
566+
{C19D9AC1-97DD-8E65-E8DB-D295A095AA2D}.Release|x64.ActiveCfg = Release|Any CPU
567+
{C19D9AC1-97DD-8E65-E8DB-D295A095AA2D}.Release|x64.Build.0 = Release|Any CPU
568+
{B268E2F0-060F-8466-7D81-ABA4D735CA59}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
569+
{B268E2F0-060F-8466-7D81-ABA4D735CA59}.Debug|Any CPU.Build.0 = Debug|Any CPU
570+
{B268E2F0-060F-8466-7D81-ABA4D735CA59}.Debug|x64.ActiveCfg = Debug|Any CPU
571+
{B268E2F0-060F-8466-7D81-ABA4D735CA59}.Debug|x64.Build.0 = Debug|Any CPU
572+
{B268E2F0-060F-8466-7D81-ABA4D735CA59}.Release|Any CPU.ActiveCfg = Release|Any CPU
573+
{B268E2F0-060F-8466-7D81-ABA4D735CA59}.Release|Any CPU.Build.0 = Release|Any CPU
574+
{B268E2F0-060F-8466-7D81-ABA4D735CA59}.Release|x64.ActiveCfg = Release|Any CPU
575+
{B268E2F0-060F-8466-7D81-ABA4D735CA59}.Release|x64.Build.0 = Release|Any CPU
576+
{970BE341-9AC8-99A5-6572-E703C1E02FCB}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
577+
{970BE341-9AC8-99A5-6572-E703C1E02FCB}.Debug|Any CPU.Build.0 = Debug|Any CPU
578+
{970BE341-9AC8-99A5-6572-E703C1E02FCB}.Debug|x64.ActiveCfg = Debug|Any CPU
579+
{970BE341-9AC8-99A5-6572-E703C1E02FCB}.Debug|x64.Build.0 = Debug|Any CPU
580+
{970BE341-9AC8-99A5-6572-E703C1E02FCB}.Release|Any CPU.ActiveCfg = Release|Any CPU
581+
{970BE341-9AC8-99A5-6572-E703C1E02FCB}.Release|Any CPU.Build.0 = Release|Any CPU
582+
{970BE341-9AC8-99A5-6572-E703C1E02FCB}.Release|x64.ActiveCfg = Release|Any CPU
583+
{970BE341-9AC8-99A5-6572-E703C1E02FCB}.Release|x64.Build.0 = Release|Any CPU
584+
{7D0DB012-9798-4BB9-B15B-A5B0B7B3B094}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
585+
{7D0DB012-9798-4BB9-B15B-A5B0B7B3B094}.Debug|Any CPU.Build.0 = Debug|Any CPU
586+
{7D0DB012-9798-4BB9-B15B-A5B0B7B3B094}.Debug|x64.ActiveCfg = Debug|Any CPU
587+
{7D0DB012-9798-4BB9-B15B-A5B0B7B3B094}.Debug|x64.Build.0 = Debug|Any CPU
588+
{7D0DB012-9798-4BB9-B15B-A5B0B7B3B094}.Release|Any CPU.ActiveCfg = Release|Any CPU
589+
{7D0DB012-9798-4BB9-B15B-A5B0B7B3B094}.Release|Any CPU.Build.0 = Release|Any CPU
590+
{7D0DB012-9798-4BB9-B15B-A5B0B7B3B094}.Release|x64.ActiveCfg = Release|Any CPU
591+
{7D0DB012-9798-4BB9-B15B-A5B0B7B3B094}.Release|x64.Build.0 = Release|Any CPU
552592
EndGlobalSection
553593
GlobalSection(SolutionProperties) = preSolution
554594
HideSolutionNode = FALSE
@@ -613,6 +653,10 @@ Global
613653
{781F1465-365C-0F22-1775-25025DAFA4C7} = {E29DC6C4-5E57-48C5-BCB0-6B8F84782749}
614654
{8D2AD45F-836A-516F-DE6A-71443CEBB18A} = {32FAFFFE-A4CB-4FEE-BF7C-84518BBC6DCC}
615655
{534D1DD5-9D62-115C-F230-47080D76CE52} = {E29DC6C4-5E57-48C5-BCB0-6B8F84782749}
656+
{C19D9AC1-97DD-8E65-E8DB-D295A095AA2D} = {32FAFFFE-A4CB-4FEE-BF7C-84518BBC6DCC}
657+
{B268E2F0-060F-8466-7D81-ABA4D735CA59} = {51AFE054-AE99-497D-A593-69BAEFB5106F}
658+
{970BE341-9AC8-99A5-6572-E703C1E02FCB} = {E29DC6C4-5E57-48C5-BCB0-6B8F84782749}
659+
{7D0DB012-9798-4BB9-B15B-A5B0B7B3B094} = {32FAFFFE-A4CB-4FEE-BF7C-84518BBC6DCC}
616660
EndGlobalSection
617661
GlobalSection(ExtensibilityGlobals) = postSolution
618662
SolutionGuid = {A9969D89-C98B-40A5-A12B-FC87E55B3A19}

‎Directory.Packages.props‎

Lines changed: 18 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -18,8 +18,10 @@
1818
<PackageVersion Include="Microsoft.Extensions.Logging" Version="9.0.0" />
1919
<PackageVersion Include="Microsoft.Extensions.Caching.Memory" Version="8.0.1" />
2020
<PackageVersion Include="Newtonsoft.Json" Version="13.0.3" />
21+
<PackageVersion Include="SharpHook" Version="5.3.8" />
22+
<PackageVersion Include="SixLabors.ImageSharp" Version="3.1.7" />
2123
<PackageVersion Include="System.ComponentModel.Annotations" Version="5.0.0" />
22-
<PackageVersion Include="System.IdentityModel.Tokens.Jwt" Version="8.0.0" />
24+
<PackageVersion Include="System.IdentityModel.Tokens.Jwt" Version="8.0.0" />
2325
<PackageVersion Include="System.Memory.Data" Version="8.0.0" />
2426
<PackageVersion Include="System.Text.Json" Version="8.0.5" />
2527
<PackageVersion Include="Serilog.Sinks.Console" Version="6.0.0" />
@@ -33,29 +35,29 @@
3335
<PackageVersion Include="Nanoid" Version="3.1.0" />
3436
<PackageVersion Include="Swashbuckle.AspNetCore" Version="6.6.2" />
3537
<PackageVersion Include="System.Security.Cryptography.Pkcs" Version="8.0.0" />
36-
<PackageVersion Include="Anthropic.SDK" Version="4.7.1" />
38+
<PackageVersion Include="Anthropic.SDK" Version="5.1.1" />
3739
<PackageVersion Include="Microsoft.AspNetCore.Mvc" Version="2.2.0" />
3840
<PackageVersion Include="NAudio" Version="2.2.1" />
3941
<PackageVersion Include="NAudio.Core" Version="2.2.1" />
40-
<PackageVersion Include="Whisper.net" Version="1.5.0" />
41-
<PackageVersion Include="Whisper.net.Runtime" Version="1.5.0" />
42+
<PackageVersion Include="Whisper.net" Version="1.8.1" />
43+
<PackageVersion Include="Whisper.net.Runtime" Version="1.8.1" />
4244
<PackageVersion Include="NCrontab" Version="3.3.3" />
43-
<PackageVersion Include="Azure.AI.OpenAI" Version="2.1.0" />
44-
<PackageVersion Include="OpenAI" Version="2.1.0" />
45-
<PackageVersion Include="MailKit" Version="4.7.0" />
45+
<PackageVersion Include="Azure.AI.OpenAI" Version="2.2.0-beta.4" />
46+
<PackageVersion Include="OpenAI" Version="2.2.0-beta.4" />
47+
<PackageVersion Include="MailKit" Version="4.11.0" />
4648
<PackageVersion Include="Microsoft.Data.Sqlite" Version="8.0.8" />
4749
<PackageVersion Include="MySql.Data" Version="9.0.0" />
4850
<PackageVersion Include="NPOI" Version="2.7.1" />
49-
<PackageVersion Include="Microsoft.AspNetCore.Mvc.Core" Version="2.2.5" />
51+
<PackageVersion Include="Microsoft.AspNetCore.Mvc.Core" Version="2.3.0" />
5052
<PackageVersion Include="Refit" Version="8.0.0" />
5153
<PackageVersion Include="Refit.HttpClientFactory" Version="8.0.0" />
5254
<PackageVersion Include="Jint" Version="4.1.0" />
5355
<PackageVersion Include="PdfPig" Version="0.1.8" />
5456
<PackageVersion Include="TensorFlow.Keras" Version="0.15.0" />
5557
<PackageVersion Include="LangChain.Providers.Google.VertexAI" Version="0.15.3-dev.58" />
5658
<PackageVersion Include="LLamaSharp" Version="0.21.0" />
57-
<PackageVersion Include="FaissMask" Version="0.2.0" />
58-
<PackageVersion Include="FastText.NetWrapper" Version="1.3.0" />
59+
<PackageVersion Include="FaissMask" Version="0.4.2" />
60+
<PackageVersion Include="FastText.NetWrapper" Version="1.3.1" />
5961
<PackageVersion Include="Microsoft.Extensions.AI.Abstractions" Version="9.3.0-preview.1.25161.3" />
6062
<PackageVersion Include="System.Text.Encodings.Web" Version="8.0.0" />
6163
<PackageVersion Include="MongoDB.Driver" Version="3.1.0" />
@@ -67,23 +69,23 @@
6769
<PackageVersion Include="Sdcb.PaddleInference.runtime.win64.mkl" Version="2.5.1" />
6870
<PackageVersion Include="Sdcb.PaddleOCR" Version="2.7.0.1" />
6971
<PackageVersion Include="Sdcb.PaddleOCR.Models.LocalV3" Version="2.7.0.1" />
70-
<PackageVersion Include="System.Drawing.Common" Version="8.0.7" />
72+
<PackageVersion Include="System.Drawing.Common" Version="8.0.14" />
7173
<PackageVersion Include="pythonnet" Version="3.0.4" />
7274
<PackageVersion Include="Qdrant.Client" Version="1.13.0" />
7375
<PackageVersion Include="Selenium.WebDriver" Version="4.27.0" />
74-
<PackageVersion Include="HtmlAgilityPack" Version="1.11.71" />
76+
<PackageVersion Include="HtmlAgilityPack" Version="1.12.0" />
7577
<PackageVersion Include="Microsoft.SemanticKernel.Abstractions" Version="1.16.0" />
7678
<PackageVersion Include="Microsoft.SemanticKernel.Plugins.Memory" Version="1.16.0-alpha" />
7779
<PackageVersion Include="Microsoft.VisualStudio.Validation" Version="17.8.8" />
7880
<PackageVersion Include="Sdcb.SparkDesk" Version="3.1.0" />
79-
<PackageVersion Include="MySqlConnector" Version="2.3.7" />
80-
<PackageVersion Include="Npgsql" Version="8.0.5" />
81+
<PackageVersion Include="MySqlConnector" Version="2.4.0" />
82+
<PackageVersion Include="Npgsql" Version="8.0.7" />
8183
<PackageVersion Include="Tencent.QCloud.Cos.Sdk" Version="5.4.39" />
8284
<PackageVersion Include="StackExchange.Redis" Version="2.7.27" />
8385
<PackageVersion Include="StrongGrid" Version="0.108.0" />
8486
<PackageVersion Include="Twilio.AspNet.Common" Version="8.1.1" />
8587
<PackageVersion Include="Twilio.AspNet.Core" Version="8.1.1" />
86-
<PackageVersion Include="Microsoft.Playwright" Version="1.49.0" />
88+
<PackageVersion Include="Microsoft.Playwright" Version="1.51.0" />
8789
<PackageVersion Include="Senparc.Weixin.MP.Middleware" Version="1.2.1" />
8890
<PackageVersion Include="System.Threading.Channels" Version="8.0.0" />
8991
<PackageVersion Include="Microsoft.AspNetCore.SignalR.StackExchangeRedis" Version="8.0.8" />
@@ -142,4 +144,4 @@
142144
<PackageVersion Include="Microsoft.AspNetCore.Authentication.JwtBearer" Version="6.0.25" />
143145
<PackageVersion Include="Microsoft.AspNetCore.SpaServices.Extensions" Version="6.0.26" />
144146
</ItemGroup>
145-
</Project>
147+
</Project>

‎src/Infrastructure/BotSharp.Abstraction.ComputerUse/BotSharp.Abstraction.ComputerUse.csproj‎

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,28 @@
1+
<Project Sdk="Microsoft.NET.Sdk">
2+
3+
<PropertyGroup>
4+
<TargetFramework>$(TargetFramework)</TargetFramework>
5+
<Nullable>enable</Nullable>
6+
<LangVersion>$(LangVersion)</LangVersion>
7+
<VersionPrefix>$(BotSharpVersion)</VersionPrefix>
8+
<PackageIcon>Icon.png</PackageIcon>
9+
<GeneratePackageOnBuild>$(GeneratePackageOnBuild)</GeneratePackageOnBuild>
10+
<OutputPath>$(SolutionDir)packages</OutputPath>
11+
</PropertyGroup>
12+
13+
<ItemGroup>
14+
<None Include="..\..\..\arts\Icon.png" Link="Icon.png">
15+
<PackagePath>\</PackagePath>
16+
<Pack>True</Pack>
17+
</None>
18+
</ItemGroup>
19+
20+
<ItemGroup>
21+
<PackageReference Include="SharpHook" />
22+
</ItemGroup>
23+
24+
<ItemGroup>
25+
<ProjectReference Include="..\BotSharp.Abstraction\BotSharp.Abstraction.csproj" />
26+
</ItemGroup>
27+
28+
</Project>

‎src/Infrastructure/BotSharp.Abstraction.ComputerUse/MLTasks/IComputerUse.cs‎

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,40 @@
1+
/*****************************************************************************
2+
Copyright 2025 Written by Haiping Chen. All Rights Reserved.
3+
4+
Licensed under the Apache License, Version 2.0 (the "License");
5+
you may not use this file except in compliance with the License.
6+
You may obtain a copy of the License at
7+
8+
http://www.apache.org/licenses/LICENSE-2.0
9+
10+
Unless required by applicable law or agreed to in writing, software
11+
distributed under the License is distributed on an "AS IS" BASIS,
12+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
See the License for the specific language governing permissions and
14+
limitations under the License.
15+
******************************************************************************/
16+
17+
using System.Threading.Tasks;
18+
19+
namespace BotSharp.Abstraction.ComputerUse.MLTasks;
20+
21+
/// <summary>
22+
/// Interface for computer use
23+
/// </summary>
24+
public interface IComputerUse
25+
{
26+
/// <summary>
27+
/// Capture screen of the specified display
28+
/// </summary>
29+
/// <param name="args"></param>
30+
/// <returns></returns>
31+
Task<ComputerUseOutput> CaptureScreen(ComputerUseArgs args);
32+
33+
Task<ComputerUseOutput> MouseMove(ComputerUseArgs args);
34+
35+
Task<ComputerUseOutput> MouseClick(ComputerUseArgs args);
36+
37+
Task<ComputerUseOutput> InputText(ComputerUseArgs args);
38+
39+
Task<ComputerUseOutput> KeyPress(ComputerUseArgs args);
40+
}

‎src/Infrastructure/BotSharp.Abstraction.ComputerUse/MLTasks/IScreenshot.cs‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
namespace BotSharp.Abstraction.ComputerUse.MLTasks;
2+
3+
public interface IScreenshot
4+
{
5+
(int x, int y) GetScreenSize(int screenIndex);
6+
byte[] CaptureScreen(int displayId);
7+
}

‎src/Infrastructure/BotSharp.Abstraction.ComputerUse/Models/ComputerUseArgs.cs‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
namespace BotSharp.Abstraction.ComputerUse.Models;
2+
3+
public class ComputerUseArgs
4+
{
5+
/// <summary>
6+
/// Number of multi-screens
7+
/// </summary>
8+
public int DisplayId { get; set; }
9+
public int X { get; set; }
10+
public int Y { get; set; }
11+
public string Text { get; set; } = string.Empty;
12+
public MouseButton MouseButton { get; set; }
13+
public KeyCode KeyCode { get; set; }
14+
}

‎src/Infrastructure/BotSharp.Abstraction.ComputerUse/Models/ComputerUseOutput.cs‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
namespace BotSharp.Abstraction.ComputerUse.Models;
2+
3+
public class ComputerUseOutput
4+
{
5+
/// <summary>
6+
/// Base64 encoded image
7+
/// </summary>
8+
public string? Screenshot { get; set; }
9+
}

‎src/Infrastructure/BotSharp.Abstraction.ComputerUse/Using.cs‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
global using System.Threading.Tasks;
2+
global using BotSharp.Abstraction.ComputerUse.Models;
3+
4+
global using SharpHook.Native;

‎src/Infrastructure/BotSharp.Abstraction/MLTasks/ILlmProviderService.cs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@ public interface ILlmProviderService
66
{
77
LlmModelSetting GetSetting(string provider, string model);
88
List<string> GetProviders();
9-
LlmModelSetting GetProviderModel(string provider, string id, bool? multiModal = null, bool realTime = false, bool imageGenerate = false);
9+
LlmModelSetting GetProviderModel(string provider, string id, bool? multiModal = null, LlmModelType? modelType = null, bool imageGenerate = false);
1010
List<LlmModelSetting> GetProviderModels(string provider);
1111
List<LlmProviderSetting> GetLlmConfigs(LlmConfigOptions? options = null);
1212
}

‎src/Infrastructure/BotSharp.Abstraction/MLTasks/Settings/LlmConfigOptions.cs‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,5 @@ public class LlmConfigOptions
44
{
55
public LlmModelType? Type { get; set; }
66
public bool? MultiModal { get; set; }
7-
public bool? RealTime { get; set; }
87
public bool? ImageGeneration { get; set; }
98
}

0 commit comments

Comments
 (0)