Skip to content

Commit e1dc28b

Browse files
Oceania2018Oceania2018
committed
Image Composition
1 parent e99cfac commit e1dc28b

File tree

18 files changed

+696
-10
lines changed

18 files changed

+696
-10
lines changed

‎Directory.Packages.props‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -6,8 +6,8 @@
66
</PropertyGroup>
77
<ItemGroup>
88
<PackageVersion Include="EntityFramework" Version="6.4.4" />
9-
<PackageVersion Include="Google_GenerativeAI" Version="3.2.0" />
10-
<PackageVersion Include="Google_GenerativeAI.Live" Version="3.2.0" />
9+
<PackageVersion Include="Google_GenerativeAI" Version="3.3.0" />
10+
<PackageVersion Include="Google_GenerativeAI.Live" Version="3.3.0" />
1111
<PackageVersion Include="LLMSharp.Google.Palm" Version="1.0.2" />
1212
<PackageVersion Include="Microsoft.AspNetCore.Http.Abstractions" Version="$(AspNetCoreVersion)" />
1313
<PackageVersion Include="Microsoft.AspNetCore.StaticFiles" Version="$(AspNetCoreVersion)" />
@@ -46,7 +46,7 @@
4646
<PackageVersion Include="Whisper.net.Runtime" Version="1.8.1" />
4747
<PackageVersion Include="NCrontab" Version="3.3.3" />
4848
<PackageVersion Include="Azure.AI.OpenAI" Version="2.3.0-beta.2" />
49-
<PackageVersion Include="OpenAI" Version="2.4.0" />
49+
<PackageVersion Include="OpenAI" Version="2.5.0" />
5050
<PackageVersion Include="MailKit" Version="4.11.0" />
5151
<PackageVersion Include="Microsoft.Data.Sqlite" Version="8.0.8" />
5252
<PackageVersion Include="MySql.Data" Version="9.0.0" />

‎src/Infrastructure/BotSharp.Abstraction/Files/IFileInstructService.cs‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@ public interface IFileInstructService
1010
Task<RoleDialogModel> VaryImage(InstructFileModel image, InstructOptions? options = null);
1111
Task<RoleDialogModel> EditImage(string text, InstructFileModel image, InstructOptions? options = null);
1212
Task<RoleDialogModel> EditImage(string text, InstructFileModel image, InstructFileModel mask, InstructOptions? options = null);
13+
Task<RoleDialogModel> ComposeImages(string text, InstructFileModel[] images, InstructOptions? options = null);
1314
#endregion
1415

1516
#region Pdf

‎src/Infrastructure/BotSharp.Abstraction/MLTasks/IImageCompletion.cs‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -24,4 +24,6 @@ public interface IImageCompletion
2424
Task<RoleDialogModel> GetImageEdits(Agent agent, RoleDialogModel message, Stream image, string imageFileName);
2525

2626
Task<RoleDialogModel> GetImageEdits(Agent agent, RoleDialogModel message, Stream image, string imageFileName, Stream mask, string maskFileName);
27+
28+
Task<RoleDialogModel> GetImageComposition(Agent agent, RoleDialogModel message, Stream[] images, string[] imageFileNames);
2729
}

‎src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.Image.cs‎

Lines changed: 56 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
1-
using BotSharp.Abstraction.Instructs.Models;
21
using BotSharp.Abstraction.Instructs;
2+
using BotSharp.Abstraction.Instructs.Models;
3+
using System.IO;
34

45
namespace BotSharp.Core.Files.Services;
56

@@ -219,4 +220,58 @@ await hook.OnResponseGenerated(new InstructResponseModel
219220

220221
return message;
221222
}
223+
224+
public async Task<RoleDialogModel> ComposeImages(string text, InstructFileModel[] images, InstructOptions? options = null)
225+
{
226+
var innerAgentId = options?.AgentId ?? Guid.Empty.ToString();
227+
var instruction = await GetAgentTemplate(innerAgentId, options?.TemplateName);
228+
229+
var completion = CompletionProvider.GetImageCompletion(_services, provider: options?.Provider ?? "openai", model: options?.Model ?? "gpt-image-1-mini");
230+
231+
var streams = new List<Stream>();
232+
var fileNames = new List<string>();
233+
foreach (var image in images)
234+
{
235+
var binary = await DownloadFile(image);
236+
237+
// Convert image
238+
var converter = GetImageConverter(options?.ImageConvertProvider);
239+
if (converter != null)
240+
{
241+
binary = await converter.ConvertImage(binary);
242+
image.FileExtension = "png";
243+
}
244+
245+
var stream = binary.ToStream();
246+
streams.Add(stream);
247+
248+
var fileName = BuildFileName(image.FileName, image.FileExtension, "image", "png");
249+
fileNames.Add(fileName);
250+
}
251+
252+
var textContent = text.IfNullOrEmptyAs(instruction).IfNullOrEmptyAs(string.Empty);
253+
var message = await completion.GetImageComposition(new Agent()
254+
{
255+
Id = innerAgentId
256+
}, new RoleDialogModel(AgentRole.User, textContent), streams.ToArray(), fileNames.ToArray());
257+
258+
foreach (var stream in streams)
259+
{
260+
stream.Close();
261+
}
262+
263+
await HookEmitter.Emit<IInstructHook>(_services, async hook =>
264+
await hook.OnResponseGenerated(new InstructResponseModel
265+
{
266+
AgentId = innerAgentId,
267+
Provider = completion.Provider,
268+
Model = completion.Model,
269+
TemplateName = options?.TemplateName,
270+
UserMessage = text,
271+
SystemInstruction = instruction,
272+
CompletionText = message.Content
273+
}), innerAgentId);
274+
275+
return message;
276+
}
222277
}

‎src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.cs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -91,7 +91,7 @@ private string BuildFileName(string? name, string? extension, string defaultName
9191

9292
private IImageConverter? GetImageConverter(string? provider)
9393
{
94-
var converter = _services.GetServices<IImageConverter>().FirstOrDefault(x => x.Provider == provider);
94+
var converter = _services.GetServices<IImageConverter>().FirstOrDefault(x => x.Provider == (provider ?? "file-handler"));
9595
return converter;
9696
}
9797
#endregion

‎src/Infrastructure/BotSharp.OpenAPI/Controllers/ConversationController.cs‎

Lines changed: 36 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -294,12 +294,12 @@ public async Task<bool> DeleteConversation([FromRoute] string conversationId)
294294
}
295295

296296
[HttpDelete("/conversation/{conversationId}/message/{messageId}")]
297-
public async Task<string?> DeleteConversationMessage([FromRoute] string conversationId, [FromRoute] string messageId, [FromBody] TruncateMessageRequest request)
297+
public async Task<IActionResult> DeleteConversationMessage([FromRoute] string conversationId, [FromRoute] string messageId, [FromBody] TruncateMessageRequest request)
298298
{
299299
var conversationService = _services.GetRequiredService<IConversationService>();
300300
var newMessageId = request.isNewMessage ? Guid.NewGuid().ToString() : null;
301301
var isSuccess = await conversationService.TruncateConversation(conversationId, messageId, newMessageId);
302-
return isSuccess ? newMessageId : string.Empty;
302+
return Ok(new { Deleted = isSuccess, MessageId = isSuccess ? newMessageId : string.Empty });
303303
}
304304

305305
#region Send notification
@@ -460,6 +460,40 @@ private async Task OnReceiveToolCallIndication(string conversationId, RoleDialog
460460
#endregion
461461

462462
#region Files and attachments
463+
[HttpGet("/conversation/{conversationId}/attachments")]
464+
public List<MessageFileViewModel> ListAttachments([FromRoute] string conversationId)
465+
{
466+
var fileStorage = _services.GetRequiredService<IFileStorageService>();
467+
var dir = fileStorage.GetDirectory(conversationId);
468+
469+
// List files in the directory
470+
var files = Directory.Exists(dir)
471+
? Directory.GetFiles(dir).Select(f => new MessageFileViewModel
472+
{
473+
FileName = Path.GetFileName(f),
474+
FileExtension = Path.GetExtension(f).TrimStart('.').ToLower(),
475+
ContentType = FileUtility.GetFileContentType(f),
476+
FileDownloadUrl = $"/conversation/{conversationId}/attachments/file/{Path.GetFileName(f)}",
477+
}).ToList()
478+
: new List<MessageFileViewModel>();
479+
480+
return files;
481+
}
482+
483+
[AllowAnonymous]
484+
[HttpGet("/conversation/{conversationId}/attachments/file/{fileName}")]
485+
public IActionResult GetAttachment([FromRoute] string conversationId, [FromRoute] string fileName)
486+
{
487+
var fileStorage = _services.GetRequiredService<IFileStorageService>();
488+
var dir = fileStorage.GetDirectory(conversationId);
489+
var filePath = Path.Combine(dir, fileName);
490+
if (!System.IO.File.Exists(filePath))
491+
{
492+
return NotFound();
493+
}
494+
return BuildFileResult(filePath);
495+
}
496+
463497
[HttpPost("/conversation/{conversationId}/attachments")]
464498
public IActionResult UploadAttachments([FromRoute] string conversationId, IFormFile[] files)
465499
{

‎src/Infrastructure/BotSharp.OpenAPI/Controllers/ImageGenerationController.cs‎

Lines changed: 54 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,54 @@
1+
using BotSharp.Abstraction.Instructs.Models;
2+
using BotSharp.OpenAPI.ViewModels.Instructs;
3+
4+
namespace BotSharp.OpenAPI.Controllers;
5+
6+
[Authorize]
7+
[ApiController]
8+
public class ImageGenerationController
9+
{
10+
private readonly IServiceProvider _services;
11+
private readonly ILogger<InstructModeController> _logger;
12+
13+
public ImageGenerationController(IServiceProvider services, ILogger<InstructModeController> logger)
14+
{
15+
_services = services;
16+
_logger = logger;
17+
}
18+
19+
[HttpPost("/instruct/image-composition")]
20+
public async Task<ImageGenerationViewModel> ComposeImages([FromBody] ImageCompositionRequest request)
21+
{
22+
var fileInstruct = _services.GetRequiredService<IFileInstructService>();
23+
var state = _services.GetRequiredService<IConversationStateService>();
24+
request.States.ForEach(x => state.SetState(x.Key, x.Value, source: StateSource.External));
25+
var imageViewModel = new ImageGenerationViewModel();
26+
27+
try
28+
{
29+
if (request.Files.Length < 1)
30+
{
31+
return new ImageGenerationViewModel { Message = "No image found" };
32+
}
33+
34+
var message = await fileInstruct.ComposeImages(request.Text, request.Files, new InstructOptions
35+
{
36+
Provider = request.Provider,
37+
Model = request.Model,
38+
AgentId = request.AgentId,
39+
TemplateName = request.TemplateName,
40+
ImageConvertProvider = request.ImageConvertProvider
41+
});
42+
imageViewModel.Content = message.Content;
43+
imageViewModel.Images = message.GeneratedImages?.Select(x => ImageViewModel.ToViewModel(x)) ?? [];
44+
return imageViewModel;
45+
}
46+
catch (Exception ex)
47+
{
48+
var error = $"Error in image edit. {ex.Message}";
49+
_logger.LogError(ex, error);
50+
imageViewModel.Message = error;
51+
return imageViewModel;
52+
}
53+
}
54+
}

‎src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/Request/InstructBaseRequest.cs‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -70,6 +70,11 @@ public class ImageEditFileRequest : ImageEditRequest
7070
public InstructFileModel File { get; set; }
7171
}
7272

73+
public class ImageCompositionRequest : ImageEditRequest
74+
{
75+
[JsonPropertyName("files")]
76+
public InstructFileModel[] Files { get; set; } = [];
77+
}
7378

7479
public class ImageMaskEditRequest : InstructBaseRequest
7580
{

‎src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Image/ImageCompletionProvider.cs‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -158,5 +158,10 @@ private int GetImageCount(string count)
158158
}
159159
return retCount;
160160
}
161+
162+
public Task<RoleDialogModel> GetImageComposition(Agent agent, RoleDialogModel message, Stream[] images, string[] imageFileNames)
163+
{
164+
throw new NotImplementedException();
165+
}
161166
#endregion
162167
}

‎src/Plugins/BotSharp.Plugin.ImageHandler/BotSharp.Plugin.ImageHandler.csproj‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,10 +10,24 @@
1010
<OutputPath>$(SolutionDir)packages</OutputPath>
1111
</PropertyGroup>
1212

13+
<ItemGroup>
14+
<None Remove="data\agents\6745151e-6d46-4a02-8de4-1c4f21c7da95\functions\util-file-compose_images.json" />
15+
<None Remove="data\agents\6745151e-6d46-4a02-8de4-1c4f21c7da95\templates\util-file-compose_images.fn.liquid" />
16+
</ItemGroup>
17+
18+
<ItemGroup>
19+
<Content Include="data\agents\6745151e-6d46-4a02-8de4-1c4f21c7da95\functions\util-file-compose_images.json">
20+
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
21+
</Content>
22+
</ItemGroup>
23+
1324
<ItemGroup>
1425
<Content Include="data\agents\6745151e-6d46-4a02-8de4-1c4f21c7da95\functions\util-file-generate_image.json">
1526
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
1627
</Content>
28+
<Content Include="data\agents\6745151e-6d46-4a02-8de4-1c4f21c7da95\templates\util-file-compose_images.fn.liquid">
29+
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
30+
</Content>
1731
<Content Include="data\agents\6745151e-6d46-4a02-8de4-1c4f21c7da95\templates\util-file-generate_image.fn.liquid">
1832
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
1933
</Content>

0 commit comments

Comments
 (0)