From d4882be2d3aedcb1f489f8e34e1920a58398032e Mon Sep 17 00:00:00 2001 From: Millmer Date: Wed, 17 Sep 2025 14:33:27 +0200 Subject: [PATCH 1/4] [issue-13131] .Net: Support BinaryContent in Gemini Connector --- dotnet/samples/Concepts/README.md | 1 + .../Core/Gemini/GeminiRequestTests.cs | 27 +++++++ ...oogleAIGeminiChatCompletionServiceTests.cs | 74 +++++++++++++++++++ ...ertexAIGeminiChatCompletionServiceTests.cs | 74 +++++++++++++++++++ .../Core/Gemini/Models/GeminiRequest.cs | 37 ++++++++++ 5 files changed, 213 insertions(+) diff --git a/dotnet/samples/Concepts/README.md b/dotnet/samples/Concepts/README.md index 0facc49d7196..77fe10e7a8ab 100644 --- a/dotnet/samples/Concepts/README.md +++ b/dotnet/samples/Concepts/README.md @@ -64,6 +64,7 @@ dotnet test -l "console;verbosity=detailed" --filter "FullyQualifiedName=ChatCom - [Google_GeminiChatCompletion](https://github.com/microsoft/semantic-kernel/blob/main/dotnet/samples/Concepts/ChatCompletion/Google_GeminiChatCompletion.cs) - [Google_GeminiChatCompletionStreaming](https://github.com/microsoft/semantic-kernel/blob/main/dotnet/samples/Concepts/ChatCompletion/Google_GeminiChatCompletionStreaming.cs) - [Google_GeminiChatCompletionWithThinkingBudget](https://github.com/microsoft/semantic-kernel/blob/main/dotnet/samples/Concepts/ChatCompletion/Google_GeminiChatCompletionWithThinkingBudget.cs) +- [Google_GeminiChatCompletionWithFile.cs](https://github.com/microsoft/semantic-kernel/blob/main/dotnet/samples/Concepts/ChatCompletion/Google_GeminiChatCompletionWithFile.cs) - [Google_GeminiGetModelResult](https://github.com/microsoft/semantic-kernel/blob/main/dotnet/samples/Concepts/ChatCompletion/Google_GeminiGetModelResult.cs) - [Google_GeminiStructuredOutputs](https://github.com/microsoft/semantic-kernel/blob/main/dotnet/samples/Concepts/ChatCompletion/Google_GeminiStructuredOutputs.cs) - [Google_GeminiVision](https://github.com/microsoft/semantic-kernel/blob/main/dotnet/samples/Concepts/ChatCompletion/Google_GeminiVision.cs) diff --git a/dotnet/src/Connectors/Connectors.Google.UnitTests/Core/Gemini/GeminiRequestTests.cs b/dotnet/src/Connectors/Connectors.Google.UnitTests/Core/Gemini/GeminiRequestTests.cs index 877b80debf67..82585eae05d8 100644 --- a/dotnet/src/Connectors/Connectors.Google.UnitTests/Core/Gemini/GeminiRequestTests.cs +++ b/dotnet/src/Connectors/Connectors.Google.UnitTests/Core/Gemini/GeminiRequestTests.cs @@ -371,6 +371,33 @@ public void FromChatHistoryAudioAsAudioContentItReturnsWithChatHistory() .SequenceEqual(Convert.FromBase64String(c.Parts![0].InlineData!.InlineData)))); } + [Fact] + public void FromChatHistoryPdfAsBinaryContentItReturnsWithChatHistory() + { + // Arrange + ReadOnlyMemory pdfAsBytes = new byte[] { 0x00, 0x01, 0x02, 0x03 }; + ChatHistory chatHistory = []; + chatHistory.AddUserMessage("user-message"); + chatHistory.AddAssistantMessage("assist-message"); + chatHistory.AddUserMessage(contentItems: + [new BinaryContent(new Uri("https://example-image.com/file.pdf")) { MimeType = "application/pdf" }]); + chatHistory.AddUserMessage(contentItems: + [new BinaryContent(pdfAsBytes, "application/pdf")]); + var executionSettings = new GeminiPromptExecutionSettings(); + + // Act + var request = GeminiRequest.FromChatHistoryAndExecutionSettings(chatHistory, executionSettings); + + // Assert + Assert.Collection(request.Contents, + c => Assert.Equal(chatHistory[0].Content, c.Parts![0].Text), + c => Assert.Equal(chatHistory[1].Content, c.Parts![0].Text), + c => Assert.Equal(chatHistory[2].Items.Cast().Single().Uri, + c.Parts![0].FileData!.FileUri), + c => Assert.True(pdfAsBytes.ToArray() + .SequenceEqual(Convert.FromBase64String(c.Parts![0].InlineData!.InlineData)))); + } + [Fact] public void FromChatHistoryUnsupportedContentItThrowsNotSupportedException() { diff --git a/dotnet/src/Connectors/Connectors.Google.UnitTests/Services/GoogleAIGeminiChatCompletionServiceTests.cs b/dotnet/src/Connectors/Connectors.Google.UnitTests/Services/GoogleAIGeminiChatCompletionServiceTests.cs index 06e5361de75e..8b1ee874314b 100644 --- a/dotnet/src/Connectors/Connectors.Google.UnitTests/Services/GoogleAIGeminiChatCompletionServiceTests.cs +++ b/dotnet/src/Connectors/Connectors.Google.UnitTests/Services/GoogleAIGeminiChatCompletionServiceTests.cs @@ -5,7 +5,9 @@ using System.IO; using System.Net.Http; using System.Text; +using System.Text.Json; using System.Threading.Tasks; +using Microsoft.SemanticKernel; using Microsoft.SemanticKernel.ChatCompletion; using Microsoft.SemanticKernel.Connectors.Google; using Microsoft.SemanticKernel.Services; @@ -144,6 +146,78 @@ public async Task RequestBodyIncludesThinkingConfigWhenSetAsync(int? thinkingBud } } + [Fact] + public async Task GetChatMessageContentsAsyncThrowsExceptionWithEmptyBinaryContentAsync() + { + // Arrange + var sut = new GoogleAIGeminiChatCompletionService("gemini-2.5-pro", "key"); + + var chatHistory = new ChatHistory(); + chatHistory.AddUserMessage([new BinaryContent()]); + + // Act & Assert + await Assert.ThrowsAsync(() => sut.GetChatMessageContentsAsync(chatHistory)); + } + + [Fact] + public async Task GetChatMessageContentsThrowsExceptionUriOnlyReferenceBinaryContentAsync() + { + // Arrange + var sut = new GoogleAIGeminiChatCompletionService("gemini-2.5-pro", "key"); + + var chatHistory = new ChatHistory(); + chatHistory.AddUserMessage([new BinaryContent(new Uri("file://testfile.pdf"))]); + + // Act & Assert + await Assert.ThrowsAsync(() => sut.GetChatMessageContentsAsync(chatHistory)); + } + + [Theory] + [InlineData(true)] + [InlineData(false)] + public async Task ItSendsBinaryContentCorrectlyAsync(bool useUriData) + { + // Arrange + var sut = new GoogleAIGeminiChatCompletionService("gemini-2.5-pro", "key", httpClient: this._httpClient); + + var mimeType = "application/pdf"; + var chatHistory = new ChatHistory(); + chatHistory.AddUserMessage([ + new TextContent("What's in this file?"), + useUriData + ? new BinaryContent($"data:{mimeType};base64,{PdfBase64Data}") + : new BinaryContent(Convert.FromBase64String(PdfBase64Data), mimeType) + ]); + + // Act + await sut.GetChatMessageContentsAsync(chatHistory); + + // Assert + var actualRequestContent = Encoding.UTF8.GetString(this._messageHandlerStub.RequestContent!); + Assert.NotNull(actualRequestContent); + var optionsJson = JsonSerializer.Deserialize(actualRequestContent); + + var contents = optionsJson.GetProperty("contents"); + Assert.Equal(1, contents.GetArrayLength()); + + var parts = contents[0].GetProperty("parts"); + Assert.Equal(2, parts.GetArrayLength()); + + Assert.True(parts[0].TryGetProperty("text", out var prompt)); + Assert.Equal("What's in this file?", prompt.ToString()); + + // Check for the file data + Assert.True(parts[1].TryGetProperty("inlineData", out var inlineData)); + Assert.Equal(JsonValueKind.Object, inlineData.ValueKind); + Assert.Equal(mimeType, inlineData.GetProperty("mimeType").GetString()); + Assert.Equal(PdfBase64Data, inlineData.GetProperty("data").ToString()); + } + + /// + /// Sample PDF data URI for testing. + /// + private const string PdfBase64Data = "JVBERi0xLjQKMSAwIG9iago8PC9UeXBlIC9DYXRhbG9nCi9QYWdlcyAyIDAgUgo+PgplbmRvYmoKMiAwIG9iago8PC9UeXBlIC9QYWdlcwovS2lkcyBbMyAwIFJdCi9Db3VudCAxCj4+CmVuZG9iagozIDAgb2JqCjw8L1R5cGUgL1BhZ2UKL1BhcmVudCAyIDAgUgovTWVkaWFCb3ggWzAgMCA1OTUgODQyXQovQ29udGVudHMgNSAwIFIKL1Jlc291cmNlcyA8PC9Qcm9jU2V0IFsvUERGIC9UZXh0XQovRm9udCA8PC9GMSA0IDAgUj4+Cj4+Cj4+CmVuZG9iago0IDAgb2JqCjw8L1R5cGUgL0ZvbnQKL1N1YnR5cGUgL1R5cGUxCi9OYW1lIC9GMQovQmFzZUZvbnQgL0hlbHZldGljYQovRW5jb2RpbmcgL01hY1JvbWFuRW5jb2RpbmcKPj4KZW5kb2JqCjUgMCBvYmoKPDwvTGVuZ3RoIDUzCj4+CnN0cmVhbQpCVAovRjEgMjAgVGYKMjIwIDQwMCBUZAooRHVtbXkgUERGKSBUagpFVAplbmRzdHJlYW0KZW5kb2JqCnhyZWYKMCA2CjAwMDAwMDAwMDAgNjU1MzUgZgowMDAwMDAwMDA5IDAwMDAwIG4KMDAwMDAwMDA2MyAwMDAwMCBuCjAwMDAwMDAxMjQgMDAwMDAgbgowMDAwMDAwMjc3IDAwMDAwIG4KMDAwMDAwMDM5MiAwMDAwMCBuCnRyYWlsZXIKPDwvU2l6ZSA2Ci9Sb290IDEgMCBSCj4+CnN0YXJ0eHJlZgo0OTUKJSVFT0YK"; + public void Dispose() { this._httpClient.Dispose(); diff --git a/dotnet/src/Connectors/Connectors.Google.UnitTests/Services/VertexAIGeminiChatCompletionServiceTests.cs b/dotnet/src/Connectors/Connectors.Google.UnitTests/Services/VertexAIGeminiChatCompletionServiceTests.cs index 179705981186..b36bc32f88ac 100644 --- a/dotnet/src/Connectors/Connectors.Google.UnitTests/Services/VertexAIGeminiChatCompletionServiceTests.cs +++ b/dotnet/src/Connectors/Connectors.Google.UnitTests/Services/VertexAIGeminiChatCompletionServiceTests.cs @@ -5,7 +5,9 @@ using System.IO; using System.Net.Http; using System.Text; +using System.Text.Json; using System.Threading.Tasks; +using Microsoft.SemanticKernel; using Microsoft.SemanticKernel.ChatCompletion; using Microsoft.SemanticKernel.Connectors.Google; using Microsoft.SemanticKernel.Services; @@ -156,6 +158,78 @@ public async Task RequestBodyIncludesThinkingConfigWhenSetAsync(int? thinkingBud } } + [Fact] + public async Task GetChatMessageContentsAsyncThrowsExceptionWithEmptyBinaryContentAsync() + { + // Arrange + var sut = new VertexAIGeminiChatCompletionService("gemini-2.5-pro", "key", "location", "project"); + + var chatHistory = new ChatHistory(); + chatHistory.AddUserMessage([new BinaryContent()]); + + // Act & Assert + await Assert.ThrowsAsync(() => sut.GetChatMessageContentsAsync(chatHistory)); + } + + [Fact] + public async Task GetChatMessageContentsThrowsExceptionUriOnlyReferenceBinaryContentAsync() + { + // Arrange + var sut = new VertexAIGeminiChatCompletionService("gemini-2.5-pro", "key", "location", "project"); + + var chatHistory = new ChatHistory(); + chatHistory.AddUserMessage([new BinaryContent(new Uri("file://testfile.pdf"))]); + + // Act & Assert + await Assert.ThrowsAsync(() => sut.GetChatMessageContentsAsync(chatHistory)); + } + + [Theory] + [InlineData(true)] + [InlineData(false)] + public async Task ItSendsBinaryContentCorrectlyAsync(bool useUriData) + { + // Arrange + var sut = new VertexAIGeminiChatCompletionService("gemini-2.5-pro", "key", "location", "project", httpClient: this._httpClient); + + var mimeType = "application/pdf"; + var chatHistory = new ChatHistory(); + chatHistory.AddUserMessage([ + new TextContent("What's in this file?"), + useUriData + ? new BinaryContent($"data:{mimeType};base64,{PdfBase64Data}") + : new BinaryContent(Convert.FromBase64String(PdfBase64Data), mimeType) + ]); + + // Act + await sut.GetChatMessageContentsAsync(chatHistory); + + // Assert + var actualRequestContent = Encoding.UTF8.GetString(this._messageHandlerStub.RequestContent!); + Assert.NotNull(actualRequestContent); + var optionsJson = JsonSerializer.Deserialize(actualRequestContent); + + var contents = optionsJson.GetProperty("contents"); + Assert.Equal(1, contents.GetArrayLength()); + + var parts = contents[0].GetProperty("parts"); + Assert.Equal(2, parts.GetArrayLength()); + + Assert.True(parts[0].TryGetProperty("text", out var prompt)); + Assert.Equal("What's in this file?", prompt.ToString()); + + // Check for the file data + Assert.True(parts[1].TryGetProperty("inlineData", out var inlineData)); + Assert.Equal(JsonValueKind.Object, inlineData.ValueKind); + Assert.Equal(mimeType, inlineData.GetProperty("mimeType").GetString()); + Assert.Equal(PdfBase64Data, inlineData.GetProperty("data").ToString()); + } + + /// + /// Sample PDF data URI for testing. + /// + private const string PdfBase64Data = "JVBERi0xLjQKMSAwIG9iago8PC9UeXBlIC9DYXRhbG9nCi9QYWdlcyAyIDAgUgo+PgplbmRvYmoKMiAwIG9iago8PC9UeXBlIC9QYWdlcwovS2lkcyBbMyAwIFJdCi9Db3VudCAxCj4+CmVuZG9iagozIDAgb2JqCjw8L1R5cGUgL1BhZ2UKL1BhcmVudCAyIDAgUgovTWVkaWFCb3ggWzAgMCA1OTUgODQyXQovQ29udGVudHMgNSAwIFIKL1Jlc291cmNlcyA8PC9Qcm9jU2V0IFsvUERGIC9UZXh0XQovRm9udCA8PC9GMSA0IDAgUj4+Cj4+Cj4+CmVuZG9iago0IDAgb2JqCjw8L1R5cGUgL0ZvbnQKL1N1YnR5cGUgL1R5cGUxCi9OYW1lIC9GMQovQmFzZUZvbnQgL0hlbHZldGljYQovRW5jb2RpbmcgL01hY1JvbWFuRW5jb2RpbmcKPj4KZW5kb2JqCjUgMCBvYmoKPDwvTGVuZ3RoIDUzCj4+CnN0cmVhbQpCVAovRjEgMjAgVGYKMjIwIDQwMCBUZAooRHVtbXkgUERGKSBUagpFVAplbmRzdHJlYW0KZW5kb2JqCnhyZWYKMCA2CjAwMDAwMDAwMDAgNjU1MzUgZgowMDAwMDAwMDA5IDAwMDAwIG4KMDAwMDAwMDA2MyAwMDAwMCBuCjAwMDAwMDAxMjQgMDAwMDAgbgowMDAwMDAwMjc3IDAwMDAwIG4KMDAwMDAwMDM5MiAwMDAwMCBuCnRyYWlsZXIKPDwvU2l6ZSA2Ci9Sb290IDEgMCBSCj4+CnN0YXJ0eHJlZgo0OTUKJSVFT0YK"; + public void Dispose() { this._httpClient.Dispose(); diff --git a/dotnet/src/Connectors/Connectors.Google/Core/Gemini/Models/GeminiRequest.cs b/dotnet/src/Connectors/Connectors.Google/Core/Gemini/Models/GeminiRequest.cs index 5d4b917ee1e7..cf1f5fa26e53 100644 --- a/dotnet/src/Connectors/Connectors.Google/Core/Gemini/Models/GeminiRequest.cs +++ b/dotnet/src/Connectors/Connectors.Google/Core/Gemini/Models/GeminiRequest.cs @@ -222,6 +222,7 @@ private static List CreateGeminiParts(ChatMessageContent content) TextContent textContent => new GeminiPart { Text = textContent.Text }, ImageContent imageContent => CreateGeminiPartFromImage(imageContent), AudioContent audioContent => CreateGeminiPartFromAudio(audioContent), + BinaryContent binaryContent => CreateGeminiPartFromBinary(binaryContent), _ => throw new NotSupportedException($"Unsupported content type. {item.GetType().Name} is not supported by Gemini.") }; @@ -297,6 +298,42 @@ private static string GetMimeTypeFromAudioContent(AudioContent audioContent) ?? throw new InvalidOperationException("Audio content MimeType is empty."); } + private static GeminiPart CreateGeminiPartFromBinary(BinaryContent binaryContent) + { + // Binary data takes precedence over URI. + if (binaryContent.Data is { IsEmpty: false }) + { + return new GeminiPart + { + InlineData = new GeminiPart.InlineDataPart + { + MimeType = GetMimeTypeFromBinaryContent(binaryContent), + InlineData = Convert.ToBase64String(binaryContent.Data.Value.ToArray()) + } + }; + } + + if (binaryContent.Uri is not null) + { + return new GeminiPart + { + FileData = new GeminiPart.FileDataPart + { + MimeType = GetMimeTypeFromBinaryContent(binaryContent), + FileUri = binaryContent.Uri ?? throw new InvalidOperationException("Binary content URI is empty.") + } + }; + } + + throw new InvalidOperationException("Binary content does not contain any data or uri."); + } + + private static string GetMimeTypeFromBinaryContent(BinaryContent binaryContent) + { + return binaryContent.MimeType + ?? throw new InvalidOperationException("BinaryCon content MimeType is empty."); + } + private static void AddConfiguration(GeminiPromptExecutionSettings executionSettings, GeminiRequest request) { request.Configuration = new ConfigurationElement From c2863cbde9fd2c7b405d33be029e2ade99cd7ead Mon Sep 17 00:00:00 2001 From: Millmer Date: Wed, 17 Sep 2025 14:35:01 +0200 Subject: [PATCH 2/4] [issue-13131] .Net: Add Gemini BinaryContent Samples --- .../Google_GeminiChatCompletionWithFile.cs | 145 ++++++++++++++++++ 1 file changed, 145 insertions(+) create mode 100644 dotnet/samples/Concepts/ChatCompletion/Google_GeminiChatCompletionWithFile.cs diff --git a/dotnet/samples/Concepts/ChatCompletion/Google_GeminiChatCompletionWithFile.cs b/dotnet/samples/Concepts/ChatCompletion/Google_GeminiChatCompletionWithFile.cs new file mode 100644 index 000000000000..b62006558376 --- /dev/null +++ b/dotnet/samples/Concepts/ChatCompletion/Google_GeminiChatCompletionWithFile.cs @@ -0,0 +1,145 @@ +// Copyright (c) Microsoft. All rights reserved. + +using Microsoft.SemanticKernel; +using Microsoft.SemanticKernel.ChatCompletion; +using Resources; + +namespace ChatCompletion; + +/// +/// This sample shows how to use binary file and inline Base64 inputs, like PDFs, with Google Gemini's chat completion. +/// +public class Google_GeminiChatCompletionWithFile(ITestOutputHelper output) : BaseTest(output) +{ + [Fact] + public async Task GoogleAIChatCompletionWithLocalFile() + { + Console.WriteLine("============= Google AI - Gemini Chat Completion With Local File ============="); + + Assert.NotNull(TestConfiguration.GoogleAI.ApiKey); + Assert.NotNull(TestConfiguration.GoogleAI.Gemini.ModelId); + + Kernel kernel = Kernel.CreateBuilder() + .AddGoogleAIGeminiChatCompletion(TestConfiguration.GoogleAI.Gemini.ModelId, TestConfiguration.GoogleAI.ApiKey) + .Build(); + + var fileBytes = await EmbeddedResource.ReadAllAsync("employees.pdf"); + + var chatHistory = new ChatHistory("You are a friendly assistant."); + chatHistory.AddUserMessage( + [ + new TextContent("What's in this file?"), + new BinaryContent(fileBytes, "application/pdf") + ]); + + var chatCompletionService = kernel.GetRequiredService(); + + var reply = await chatCompletionService.GetChatMessageContentAsync(chatHistory); + + Console.WriteLine(reply.Content); + } + + [Fact] + public async Task VertexAIChatCompletionWithLocalFile() + { + Console.WriteLine("============= Vertex AI - Gemini Chat Completion With Local File ============="); + + Assert.NotNull(TestConfiguration.VertexAI.BearerKey); + Assert.NotNull(TestConfiguration.VertexAI.Location); + Assert.NotNull(TestConfiguration.VertexAI.ProjectId); + Assert.NotNull(TestConfiguration.VertexAI.Gemini.ModelId); + + Kernel kernel = Kernel.CreateBuilder() + .AddVertexAIGeminiChatCompletion( + modelId: TestConfiguration.VertexAI.Gemini.ModelId, + bearerKey: TestConfiguration.VertexAI.BearerKey, + location: TestConfiguration.VertexAI.Location, + projectId: TestConfiguration.VertexAI.ProjectId) + .Build(); + + var fileBytes = await EmbeddedResource.ReadAllAsync("employees.pdf"); + + var chatHistory = new ChatHistory("You are a friendly assistant."); + chatHistory.AddUserMessage( + [ + new TextContent("What's in this file?"), + new BinaryContent(fileBytes, "application/pdf"), + ]); + + var chatCompletionService = kernel.GetRequiredService(); + + var reply = await chatCompletionService.GetChatMessageContentAsync(chatHistory); + + Console.WriteLine(reply.Content); + } + + [Fact] + public async Task GoogleAIChatCompletionWithBase64DataUri() + { + Console.WriteLine("============= Google AI - Gemini Chat Completion With Base64 Data Uri ============="); + + Assert.NotNull(TestConfiguration.GoogleAI.ApiKey); + Assert.NotNull(TestConfiguration.GoogleAI.Gemini.ModelId); + + Kernel kernel = Kernel.CreateBuilder() + .AddGoogleAIGeminiChatCompletion(TestConfiguration.GoogleAI.Gemini.ModelId, TestConfiguration.GoogleAI.ApiKey) + .Build(); + + var fileBytes = await EmbeddedResource.ReadAllAsync("employees.pdf"); + var fileBase64 = Convert.ToBase64String(fileBytes.ToArray()); + var dataUri = $"data:application/pdf;base64,{fileBase64}"; + + var chatHistory = new ChatHistory("You are a friendly assistant."); + chatHistory.AddUserMessage( + [ + new TextContent("What's in this file?"), + new BinaryContent(dataUri) + // Google AI Gemini AI does not support arbitrary URIs but we can convert a Base64 URI into InlineData with the correct mimeType. + ]); + + var chatCompletionService = kernel.GetRequiredService(); + + var reply = await chatCompletionService.GetChatMessageContentAsync(chatHistory); + + Console.WriteLine(reply.Content); + } + + [Fact] + public async Task VertexAIChatCompletionWithBase64DataUri() + { + Console.WriteLine("============= Vertex AI - Gemini Chat Completion With Base64 Data Uri ============="); + + Assert.NotNull(TestConfiguration.VertexAI.BearerKey); + Assert.NotNull(TestConfiguration.VertexAI.Location); + Assert.NotNull(TestConfiguration.VertexAI.ProjectId); + Assert.NotNull(TestConfiguration.VertexAI.Gemini.ModelId); + + Kernel kernel = Kernel.CreateBuilder() + .AddVertexAIGeminiChatCompletion( + modelId: TestConfiguration.VertexAI.Gemini.ModelId, + bearerKey: TestConfiguration.VertexAI.BearerKey, + location: TestConfiguration.VertexAI.Location, + projectId: TestConfiguration.VertexAI.ProjectId) + .Build(); + + var fileBytes = await EmbeddedResource.ReadAllAsync("employees.pdf"); + var fileBase64 = Convert.ToBase64String(fileBytes.ToArray()); + var dataUri = $"data:application/pdf;base64,{fileBase64}"; + + var chatHistory = new ChatHistory("You are a friendly assistant."); + chatHistory.AddUserMessage( + [ + new TextContent("What's in this file?"), + new BinaryContent(dataUri) + // Vertex AI API does not support URIs outside of inline Base64 or GCS buckets within the same project. The bucket that stores the file must be in the same Google Cloud project that's sending the request. You must always provide the mimeType via the metadata property. + // var content = new BinaryContent(gs://generativeai-downloads/files/employees.pdf); + // content.Metadata = new Dictionary { { "mimeType", "application/pdf" } }; + ]); + + var chatCompletionService = kernel.GetRequiredService(); + + var reply = await chatCompletionService.GetChatMessageContentAsync(chatHistory); + + Console.WriteLine(reply.Content); + } +} From 014e83e11020ad9a02d491cb06191f9ae4fe07e9 Mon Sep 17 00:00:00 2001 From: Millmer Date: Thu, 18 Sep 2025 09:26:58 +0200 Subject: [PATCH 3/4] chore: fix typos --- .../Core/Gemini/GeminiRequestTests.cs | 2 +- .../Connectors.Google/Core/Gemini/Models/GeminiRequest.cs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/dotnet/src/Connectors/Connectors.Google.UnitTests/Core/Gemini/GeminiRequestTests.cs b/dotnet/src/Connectors/Connectors.Google.UnitTests/Core/Gemini/GeminiRequestTests.cs index 82585eae05d8..fc85828c7405 100644 --- a/dotnet/src/Connectors/Connectors.Google.UnitTests/Core/Gemini/GeminiRequestTests.cs +++ b/dotnet/src/Connectors/Connectors.Google.UnitTests/Core/Gemini/GeminiRequestTests.cs @@ -380,7 +380,7 @@ public void FromChatHistoryPdfAsBinaryContentItReturnsWithChatHistory() chatHistory.AddUserMessage("user-message"); chatHistory.AddAssistantMessage("assist-message"); chatHistory.AddUserMessage(contentItems: - [new BinaryContent(new Uri("https://example-image.com/file.pdf")) { MimeType = "application/pdf" }]); + [new BinaryContent(new Uri("https://example-file.com/file.pdf")) { MimeType = "application/pdf" }]); chatHistory.AddUserMessage(contentItems: [new BinaryContent(pdfAsBytes, "application/pdf")]); var executionSettings = new GeminiPromptExecutionSettings(); diff --git a/dotnet/src/Connectors/Connectors.Google/Core/Gemini/Models/GeminiRequest.cs b/dotnet/src/Connectors/Connectors.Google/Core/Gemini/Models/GeminiRequest.cs index cf1f5fa26e53..051d3ec8ba5d 100644 --- a/dotnet/src/Connectors/Connectors.Google/Core/Gemini/Models/GeminiRequest.cs +++ b/dotnet/src/Connectors/Connectors.Google/Core/Gemini/Models/GeminiRequest.cs @@ -331,7 +331,7 @@ private static GeminiPart CreateGeminiPartFromBinary(BinaryContent binaryContent private static string GetMimeTypeFromBinaryContent(BinaryContent binaryContent) { return binaryContent.MimeType - ?? throw new InvalidOperationException("BinaryCon content MimeType is empty."); + ?? throw new InvalidOperationException("Binary content MimeType is empty."); } private static void AddConfiguration(GeminiPromptExecutionSettings executionSettings, GeminiRequest request) From 1eddbedcb017de376cdb00f67cb035bb0abf949f Mon Sep 17 00:00:00 2001 From: Millmer Date: Sat, 20 Sep 2025 11:36:58 +0200 Subject: [PATCH 4/4] [issue-13131] .Net: Add Gemini BinaryContent Integration Test --- .../Gemini/GeminiChatCompletionTests.cs | 26 +++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/dotnet/src/IntegrationTests/Connectors/Google/Gemini/GeminiChatCompletionTests.cs b/dotnet/src/IntegrationTests/Connectors/Google/Gemini/GeminiChatCompletionTests.cs index 7645d9cf107e..9ea931ef4638 100644 --- a/dotnet/src/IntegrationTests/Connectors/Google/Gemini/GeminiChatCompletionTests.cs +++ b/dotnet/src/IntegrationTests/Connectors/Google/Gemini/GeminiChatCompletionTests.cs @@ -381,6 +381,32 @@ public async Task ChatGenerationAudioUriAsync(ServiceType serviceType) Assert.Contains("brooklyn bridge", response.Content, StringComparison.OrdinalIgnoreCase); } + [RetryTheory] + [InlineData(ServiceType.GoogleAI, Skip = "This test is for manual verification.")] + [InlineData(ServiceType.VertexAI, Skip = "This test is for manual verification.")] + public async Task ChatGenerationWithBinaryFileDataAsync(ServiceType serviceType) + { + // Arrange + Memory file = await File.ReadAllBytesAsync(Path.Combine("TestData", "employees.pdf")); + var chatHistory = new ChatHistory(); + var messageContent = new ChatMessageContent(AuthorRole.User, items: + [ + new TextContent("What positions do the employees have?"), + new BinaryContent(file, "application/pdf") + ]); + chatHistory.Add(messageContent); + + var sut = this.GetChatService(serviceType); + + // Act + var response = await sut.GetChatMessageContentAsync(chatHistory); + + // Assert + Assert.NotNull(response.Content); + this.Output.WriteLine(response.Content); + Assert.Contains("accountant", response.Content, StringComparison.OrdinalIgnoreCase); + } + [RetryTheory] [InlineData(ServiceType.GoogleAI, Skip = "Currently GoogleAI always returns zero tokens.")] [InlineData(ServiceType.VertexAI, Skip = "This test is for manual verification.")]