diff --git a/dotnet/samples/Concepts/Search/Bing_TextSearch.cs b/dotnet/samples/Concepts/Search/Bing_TextSearch.cs index 78e0af672036..b862360d740c 100644 --- a/dotnet/samples/Concepts/Search/Bing_TextSearch.cs +++ b/dotnet/samples/Concepts/Search/Bing_TextSearch.cs @@ -119,6 +119,86 @@ public async Task UsingBingTextSearchWithASiteFilterAsync() } } + /// + /// Show how to use enhanced LINQ filtering with BingTextSearch for type-safe searches. + /// + [Fact] + public async Task UsingBingTextSearchWithLinqFilteringAsync() + { + // Create a logging handler to output HTTP requests and responses + LoggingHandler handler = new(new HttpClientHandler(), this.Output); + using HttpClient httpClient = new(handler); + + // Create an ITextSearch instance for type-safe LINQ filtering + ITextSearch textSearch = new BingTextSearch(apiKey: TestConfiguration.Bing.ApiKey, options: new() { HttpClient = httpClient }); + + var query = "Semantic Kernel AI"; + + // Example 1: Filter by language (English only) + Console.WriteLine("——— Example 1: Language Filter (English) ———\n"); + var languageOptions = new TextSearchOptions + { + Top = 2, + Filter = page => page.Language == "en" + }; + var languageResults = await textSearch.SearchAsync(query, languageOptions); + await foreach (string result in languageResults.Results) + { + Console.WriteLine(result); + WriteHorizontalRule(); + } + + // Example 2: Filter by family-friendly content + Console.WriteLine("\n——— Example 2: Family Friendly Filter ———\n"); + var familyFriendlyOptions = new TextSearchOptions + { + Top = 2, + Filter = page => page.IsFamilyFriendly == true + }; + var familyFriendlyResults = await textSearch.SearchAsync(query, familyFriendlyOptions); + await foreach (string result in familyFriendlyResults.Results) + { + Console.WriteLine(result); + WriteHorizontalRule(); + } + + // Example 3: Compound AND filtering (language + family-friendly) + Console.WriteLine("\n——— Example 3: Compound Filter (English + Family Friendly) ———\n"); + var compoundOptions = new TextSearchOptions + { + Top = 2, + Filter = page => page.Language == "en" && page.IsFamilyFriendly == true + }; + var compoundResults = await textSearch.GetSearchResultsAsync(query, compoundOptions); + await foreach (BingWebPage page in compoundResults.Results) + { + Console.WriteLine($"Name: {page.Name}"); + Console.WriteLine($"Snippet: {page.Snippet}"); + Console.WriteLine($"Language: {page.Language}"); + Console.WriteLine($"Family Friendly: {page.IsFamilyFriendly}"); + WriteHorizontalRule(); + } + + // Example 4: Complex compound filtering with nullable checks + Console.WriteLine("\n——— Example 4: Complex Compound Filter (Language + Site + Family Friendly) ———\n"); + var complexOptions = new TextSearchOptions + { + Top = 2, + Filter = page => page.Language == "en" && + page.IsFamilyFriendly == true && + page.DisplayUrl != null && page.DisplayUrl.Contains("microsoft") + }; + var complexResults = await textSearch.GetSearchResultsAsync(query, complexOptions); + await foreach (BingWebPage page in complexResults.Results) + { + Console.WriteLine($"Name: {page.Name}"); + Console.WriteLine($"Display URL: {page.DisplayUrl}"); + Console.WriteLine($"Language: {page.Language}"); + Console.WriteLine($"Family Friendly: {page.IsFamilyFriendly}"); + WriteHorizontalRule(); + } + } + #region private /// /// Test mapper which converts an arbitrary search result to a string using JSON serialization. diff --git a/dotnet/samples/Concepts/Search/Google_TextSearch.cs b/dotnet/samples/Concepts/Search/Google_TextSearch.cs index a77f65bcfbc3..749405422faf 100644 --- a/dotnet/samples/Concepts/Search/Google_TextSearch.cs +++ b/dotnet/samples/Concepts/Search/Google_TextSearch.cs @@ -26,7 +26,7 @@ public async Task UsingGoogleTextSearchAsync() var query = "What is the Semantic Kernel?"; // Search and return results as string items - KernelSearchResults stringResults = await textSearch.SearchAsync(query, new() { Top = 4, Skip = 0 }); + KernelSearchResults stringResults = await textSearch.SearchAsync(query, new TextSearchOptions { Top = 4, Skip = 0 }); Console.WriteLine("——— String Results ———\n"); await foreach (string result in stringResults.Results) { @@ -35,7 +35,7 @@ public async Task UsingGoogleTextSearchAsync() } // Search and return results as TextSearchResult items - KernelSearchResults textResults = await textSearch.GetTextSearchResultsAsync(query, new() { Top = 4, Skip = 4 }); + KernelSearchResults textResults = await textSearch.GetTextSearchResultsAsync(query, new TextSearchOptions { Top = 4, Skip = 4 }); Console.WriteLine("\n——— Text Search Results ———\n"); await foreach (TextSearchResult result in textResults.Results) { @@ -46,7 +46,7 @@ public async Task UsingGoogleTextSearchAsync() } // Search and return results as Google.Apis.CustomSearchAPI.v1.Data.Result items - KernelSearchResults fullResults = await textSearch.GetSearchResultsAsync(query, new() { Top = 4, Skip = 8 }); + KernelSearchResults fullResults = await textSearch.GetSearchResultsAsync(query, new TextSearchOptions { Top = 4, Skip = 8 }); Console.WriteLine("\n——— Google Web Page Results ———\n"); await foreach (Google.Apis.CustomSearchAPI.v1.Data.Result result in fullResults.Results) { @@ -74,7 +74,7 @@ public async Task UsingGoogleTextSearchWithACustomMapperAsync() var query = "What is the Semantic Kernel?"; // Search with TextSearchResult textResult type - KernelSearchResults stringResults = await textSearch.SearchAsync(query, new() { Top = 2, Skip = 0 }); + KernelSearchResults stringResults = await textSearch.SearchAsync(query, new TextSearchOptions { Top = 2, Skip = 0 }); Console.WriteLine("--- Serialized JSON Results ---"); await foreach (string result in stringResults.Results) { @@ -107,6 +107,113 @@ public async Task UsingGoogleTextSearchWithASiteSearchFilterAsync() } } + /// + /// Show how to use enhanced LINQ filtering with GoogleTextSearch including Contains, NOT, FileType, and compound AND expressions. + /// + [Fact] + public async Task UsingGoogleTextSearchWithEnhancedLinqFilteringAsync() + { + // Create an ITextSearch instance using Google search + var textSearch = new GoogleTextSearch( + initializer: new() { ApiKey = TestConfiguration.Google.ApiKey, HttpClientFactory = new CustomHttpClientFactory(this.Output) }, + searchEngineId: TestConfiguration.Google.SearchEngineId); + + var query = "Semantic Kernel AI"; + + // Example 1: Simple equality filtering + Console.WriteLine("——— Example 1: Equality Filter (DisplayLink) ———\n"); + var equalityOptions = new TextSearchOptions + { + Top = 2, + Skip = 0, + Filter = page => page.DisplayLink == "microsoft.com" + }; + var equalityResults = await textSearch.SearchAsync(query, equalityOptions); + await foreach (string result in equalityResults.Results) + { + Console.WriteLine(result); + Console.WriteLine(new string('—', HorizontalRuleLength)); + } + + // Example 2: Contains filtering + Console.WriteLine("\n——— Example 2: Contains Filter (Title) ———\n"); + var containsOptions = new TextSearchOptions + { + Top = 2, + Skip = 0, + Filter = page => page.Title != null && page.Title.Contains("AI") + }; + var containsResults = await textSearch.SearchAsync(query, containsOptions); + await foreach (string result in containsResults.Results) + { + Console.WriteLine(result); + Console.WriteLine(new string('—', HorizontalRuleLength)); + } + + // Example 3: NOT Contains filtering (exclusion) + Console.WriteLine("\n——— Example 3: NOT Contains Filter (Exclude 'deprecated') ———\n"); + var notContainsOptions = new TextSearchOptions + { + Top = 2, + Skip = 0, + Filter = page => page.Title != null && !page.Title.Contains("deprecated") + }; + var notContainsResults = await textSearch.SearchAsync(query, notContainsOptions); + await foreach (string result in notContainsResults.Results) + { + Console.WriteLine(result); + Console.WriteLine(new string('—', HorizontalRuleLength)); + } + + // Example 4: FileFormat filtering + Console.WriteLine("\n——— Example 4: FileFormat Filter (PDF files) ———\n"); + var fileFormatOptions = new TextSearchOptions + { + Top = 2, + Skip = 0, + Filter = page => page.FileFormat == "pdf" + }; + var fileFormatResults = await textSearch.SearchAsync(query, fileFormatOptions); + await foreach (string result in fileFormatResults.Results) + { + Console.WriteLine(result); + Console.WriteLine(new string('—', HorizontalRuleLength)); + } + + // Example 5: Compound AND filtering (multiple conditions) + Console.WriteLine("\n——— Example 5: Compound AND Filter (Title + Site) ———\n"); + var compoundOptions = new TextSearchOptions + { + Top = 2, + Skip = 0, + Filter = page => page.Title != null && page.Title.Contains("Semantic") && + page.DisplayLink != null && page.DisplayLink.Contains("microsoft") + }; + var compoundResults = await textSearch.SearchAsync(query, compoundOptions); + await foreach (string result in compoundResults.Results) + { + Console.WriteLine(result); + Console.WriteLine(new string('—', HorizontalRuleLength)); + } + + // Example 6: Complex compound filtering (equality + contains + exclusion) + Console.WriteLine("\n——— Example 6: Complex Compound Filter (FileFormat + Contains + NOT Contains) ———\n"); + var complexOptions = new TextSearchOptions + { + Top = 2, + Skip = 0, + Filter = page => page.FileFormat == "pdf" && + page.Title != null && page.Title.Contains("AI") && + page.Snippet != null && !page.Snippet.Contains("deprecated") + }; + var complexResults = await textSearch.SearchAsync(query, complexOptions); + await foreach (string result in complexResults.Results) + { + Console.WriteLine(result); + Console.WriteLine(new string('—', HorizontalRuleLength)); + } + } + #region private private const int HorizontalRuleLength = 80; diff --git a/dotnet/samples/Concepts/Search/Tavily_TextSearch.cs b/dotnet/samples/Concepts/Search/Tavily_TextSearch.cs index 18078eaef238..82161b28dd63 100644 --- a/dotnet/samples/Concepts/Search/Tavily_TextSearch.cs +++ b/dotnet/samples/Concepts/Search/Tavily_TextSearch.cs @@ -182,6 +182,86 @@ public async Task UsingTavilyTextSearchWithAnIncludeDomainFilterAsync() } } + /// + /// Show how to use enhanced LINQ filtering with TavilyTextSearch for type-safe searches with Title.Contains() support. + /// + [Fact] + public async Task UsingTavilyTextSearchWithLinqFilteringAsync() + { + // Create a logging handler to output HTTP requests and responses + LoggingHandler handler = new(new HttpClientHandler(), this.Output); + using HttpClient httpClient = new(handler); + + // Create an ITextSearch instance for type-safe LINQ filtering + ITextSearch textSearch = new TavilyTextSearch(apiKey: TestConfiguration.Tavily.ApiKey, options: new() { HttpClient = httpClient }); + + var query = "Semantic Kernel AI"; + + // Example 1: Filter results by title content using Contains + Console.WriteLine("——— Example 1: Title Contains Filter ———\n"); + var titleContainsOptions = new TextSearchOptions + { + Top = 2, + Filter = page => page.Title != null && page.Title.Contains("Kernel") + }; + var titleResults = await textSearch.SearchAsync(query, titleContainsOptions); + await foreach (string result in titleResults.Results) + { + Console.WriteLine(result); + WriteHorizontalRule(); + } + + // Example 2: Compound AND filtering (title contains + NOT contains) + Console.WriteLine("\n——— Example 2: Compound Filter (Title Contains + Exclusion) ———\n"); + var compoundOptions = new TextSearchOptions + { + Top = 2, + Filter = page => page.Title != null && page.Title.Contains("AI") && + page.Content != null && !page.Content.Contains("deprecated") + }; + var compoundResults = await textSearch.SearchAsync(query, compoundOptions); + await foreach (string result in compoundResults.Results) + { + Console.WriteLine(result); + WriteHorizontalRule(); + } + + // Example 3: Get full results with LINQ filtering + Console.WriteLine("\n——— Example 3: Full Results with Title Filter ———\n"); + var fullResultsOptions = new TextSearchOptions + { + Top = 2, + Filter = page => page.Title != null && page.Title.Contains("Semantic") + }; + var fullResults = await textSearch.GetSearchResultsAsync(query, fullResultsOptions); + await foreach (TavilyWebPage page in fullResults.Results) + { + Console.WriteLine($"Title: {page.Title}"); + Console.WriteLine($"Content: {page.Content}"); + Console.WriteLine($"URL: {page.Url}"); + Console.WriteLine($"Score: {page.Score}"); + WriteHorizontalRule(); + } + + // Example 4: Complex compound filtering with multiple conditions + Console.WriteLine("\n——— Example 4: Complex Compound Filter (Title + Content + URL) ———\n"); + var complexOptions = new TextSearchOptions + { + Top = 2, + Filter = page => page.Title != null && page.Title.Contains("Kernel") && + page.Content != null && page.Content.Contains("AI") && + page.Url != null && page.Url.ToString().Contains("microsoft") + }; + var complexResults = await textSearch.GetSearchResultsAsync(query, complexOptions); + await foreach (TavilyWebPage page in complexResults.Results) + { + Console.WriteLine($"Title: {page.Title}"); + Console.WriteLine($"URL: {page.Url}"); + Console.WriteLine($"Score: {page.Score}"); + WriteHorizontalRule(); + } + } + #region private /// /// Test mapper which converts an arbitrary search result to a string using JSON serialization. diff --git a/dotnet/samples/GettingStartedWithTextSearch/InMemoryVectorStoreFixture.cs b/dotnet/samples/GettingStartedWithTextSearch/InMemoryVectorStoreFixture.cs index 02d54e8367a3..c409d53b8260 100644 --- a/dotnet/samples/GettingStartedWithTextSearch/InMemoryVectorStoreFixture.cs +++ b/dotnet/samples/GettingStartedWithTextSearch/InMemoryVectorStoreFixture.cs @@ -15,12 +15,24 @@ namespace GettingStartedWithTextSearch; /// public class InMemoryVectorStoreFixture : IAsyncLifetime { + /// + /// Gets the embedding generator used for creating vector embeddings. + /// public IEmbeddingGenerator> EmbeddingGenerator { get; private set; } + /// + /// Gets the in-memory vector store instance. + /// public InMemoryVectorStore InMemoryVectorStore { get; private set; } + /// + /// Gets the vector store record collection for data models. + /// public VectorStoreCollection VectorStoreRecordCollection { get; private set; } + /// + /// Gets the name of the collection used for storing records. + /// public string CollectionName => "records"; /// @@ -138,21 +150,36 @@ private async Task> CreateCollectionFromLis /// public sealed class DataModel { + /// + /// Gets or sets the unique identifier for this record. + /// [VectorStoreKey] [TextSearchResultName] public Guid Key { get; init; } + /// + /// Gets or sets the text content of this record. + /// [VectorStoreData] [TextSearchResultValue] public string Text { get; init; } + /// + /// Gets or sets the link associated with this record. + /// [VectorStoreData] [TextSearchResultLink] public string Link { get; init; } + /// + /// Gets or sets the tag for categorizing this record. + /// [VectorStoreData(IsIndexed = true)] public required string Tag { get; init; } + /// + /// Gets the embedding representation of the text content. + /// [VectorStoreVector(1536)] public string Embedding => Text; } diff --git a/dotnet/samples/GettingStartedWithTextSearch/Step1_Web_Search.cs b/dotnet/samples/GettingStartedWithTextSearch/Step1_Web_Search.cs index a5676b9f1c5d..b3143f23c307 100644 --- a/dotnet/samples/GettingStartedWithTextSearch/Step1_Web_Search.cs +++ b/dotnet/samples/GettingStartedWithTextSearch/Step1_Web_Search.cs @@ -1,5 +1,7 @@ // Copyright (c) Microsoft. All rights reserved. +#pragma warning disable CS0618 // ITextSearch is obsolete - Sample demonstrates legacy interface usage + using Microsoft.SemanticKernel.Data; using Microsoft.SemanticKernel.Plugins.Web.Bing; using Microsoft.SemanticKernel.Plugins.Web.Google; @@ -23,7 +25,7 @@ public async Task BingSearchAsync() var query = "What is the Semantic Kernel?"; // Search and return results - KernelSearchResults searchResults = await textSearch.SearchAsync(query, new() { Top = 4 }); + KernelSearchResults searchResults = await textSearch.SearchAsync(query, new TextSearchOptions { Top = 4 }); await foreach (string result in searchResults.Results) { Console.WriteLine(result); @@ -44,13 +46,84 @@ public async Task GoogleSearchAsync() var query = "What is the Semantic Kernel?"; // Search and return results - KernelSearchResults searchResults = await textSearch.SearchAsync(query, new() { Top = 4 }); + KernelSearchResults searchResults = await textSearch.SearchAsync(query, new TextSearchOptions { Top = 4 }); await foreach (string result in searchResults.Results) { Console.WriteLine(result); } } + /// + /// Show how to use with the new generic + /// interface and LINQ filtering for type-safe searches. + /// + [Fact] + public async Task BingSearchWithLinqFilteringAsync() + { +#pragma warning disable CA1859 // Use concrete types when possible for improved performance - Sample intentionally demonstrates interface usage + // Create an ITextSearch instance for type-safe LINQ filtering + ITextSearch textSearch = new BingTextSearch(apiKey: TestConfiguration.Bing.ApiKey); +#pragma warning restore CA1859 + + var query = "What is the Semantic Kernel?"; + + // Use LINQ filtering for type-safe search with compile-time validation + var options = new TextSearchOptions + { + Top = 4, + Filter = page => page.Language == "en" && page.IsFamilyFriendly == true + }; + + // Search and return strongly-typed results + Console.WriteLine("\n--- Bing Search with LINQ Filtering ---\n"); + KernelSearchResults searchResults = await textSearch.GetSearchResultsAsync(query, options); + await foreach (BingWebPage page in searchResults.Results) + { + Console.WriteLine($"Name: {page.Name}"); + Console.WriteLine($"Snippet: {page.Snippet}"); + Console.WriteLine($"Url: {page.Url}"); + Console.WriteLine($"Language: {page.Language}"); + Console.WriteLine($"Family Friendly: {page.IsFamilyFriendly}"); + Console.WriteLine("---"); + } + } + + /// + /// Show how to use with the new generic + /// interface and LINQ filtering for type-safe searches. + /// + [Fact] + public async Task GoogleSearchWithLinqFilteringAsync() + { +#pragma warning disable CA1859 // Use concrete types when possible for improved performance - Sample intentionally demonstrates interface usage + // Create an ITextSearch instance for type-safe LINQ filtering + ITextSearch textSearch = new GoogleTextSearch( + searchEngineId: TestConfiguration.Google.SearchEngineId, + apiKey: TestConfiguration.Google.ApiKey); +#pragma warning restore CA1859 + + var query = "What is the Semantic Kernel?"; + + // Use LINQ filtering for type-safe search with compile-time validation + var options = new TextSearchOptions + { + Top = 4, + Filter = page => page.Title != null && page.Title.Contains("Semantic") && page.DisplayLink != null && page.DisplayLink.EndsWith(".com") + }; + + // Search and return strongly-typed results + Console.WriteLine("\n--- Google Search with LINQ Filtering ---\n"); + KernelSearchResults searchResults = await textSearch.GetSearchResultsAsync(query, options); + await foreach (GoogleWebPage page in searchResults.Results) + { + Console.WriteLine($"Title: {page.Title}"); + Console.WriteLine($"Snippet: {page.Snippet}"); + Console.WriteLine($"Link: {page.Link}"); + Console.WriteLine($"Display Link: {page.DisplayLink}"); + Console.WriteLine("---"); + } + } + /// /// Show how to create a and use it to perform a search /// and return results as a collection of instances. @@ -84,7 +157,7 @@ public async Task SearchForWebPagesAsync() } else { - Console.WriteLine("\n——— Google Web Page Results ———\n"); + Console.WriteLine("\n--- Google Web Page Results ---\n"); await foreach (Google.Apis.CustomSearchAPI.v1.Data.Result result in objectResults.Results) { Console.WriteLine($"Title: {result.Title}"); diff --git a/dotnet/samples/GettingStartedWithTextSearch/Step2_Search_For_RAG.cs b/dotnet/samples/GettingStartedWithTextSearch/Step2_Search_For_RAG.cs index cb21cccc66b4..1278f8a59141 100644 --- a/dotnet/samples/GettingStartedWithTextSearch/Step2_Search_For_RAG.cs +++ b/dotnet/samples/GettingStartedWithTextSearch/Step2_Search_For_RAG.cs @@ -1,4 +1,7 @@ // Copyright (c) Microsoft. All rights reserved. + +#pragma warning disable CS0618 // ITextSearch is obsolete - Sample demonstrates legacy interface usage + using System.Text.RegularExpressions; using HtmlAgilityPack; using Microsoft.SemanticKernel; diff --git a/dotnet/src/IntegrationTests/Agents/CommonInterfaceConformance/AgentWithTextSearchProviderConformance/AgentWithTextSearchProvider.cs b/dotnet/src/IntegrationTests/Agents/CommonInterfaceConformance/AgentWithTextSearchProviderConformance/AgentWithTextSearchProvider.cs index 4d350564b7de..89e0a1790648 100644 --- a/dotnet/src/IntegrationTests/Agents/CommonInterfaceConformance/AgentWithTextSearchProviderConformance/AgentWithTextSearchProvider.cs +++ b/dotnet/src/IntegrationTests/Agents/CommonInterfaceConformance/AgentWithTextSearchProviderConformance/AgentWithTextSearchProvider.cs @@ -41,7 +41,9 @@ public abstract class AgentWithTextSearchProvider(Func creat public async Task TextSearchBehaviorStateIsUsedByAgentInternalAsync(string question, string expectedResult, params string[] ragResults) { // Arrange +#pragma warning disable CS0618 // ITextSearch is obsolete - Testing legacy interface var mockTextSearch = new Mock(); +#pragma warning restore CS0618 mockTextSearch.Setup(x => x.GetTextSearchResultsAsync( It.IsAny(), It.IsAny(), diff --git a/dotnet/src/IntegrationTests/Connectors/Memory/AzureAISearch/AzureAISearchTextSearchTests.cs b/dotnet/src/IntegrationTests/Connectors/Memory/AzureAISearch/AzureAISearchTextSearchTests.cs index aaf65fa5cb4a..9280df1f513c 100644 --- a/dotnet/src/IntegrationTests/Connectors/Memory/AzureAISearch/AzureAISearchTextSearchTests.cs +++ b/dotnet/src/IntegrationTests/Connectors/Memory/AzureAISearch/AzureAISearchTextSearchTests.cs @@ -1,5 +1,7 @@ // Copyright (c) Microsoft. All rights reserved. +#pragma warning disable CS0618 // ITextSearch is obsolete + using System; using System.Threading.Tasks; using Azure.AI.OpenAI; diff --git a/dotnet/src/IntegrationTests/Connectors/Memory/InMemory/InMemoryVectorStoreTextSearchTests.cs b/dotnet/src/IntegrationTests/Connectors/Memory/InMemory/InMemoryVectorStoreTextSearchTests.cs index a5f6c4e6ec4c..cf41f187dca3 100644 --- a/dotnet/src/IntegrationTests/Connectors/Memory/InMemory/InMemoryVectorStoreTextSearchTests.cs +++ b/dotnet/src/IntegrationTests/Connectors/Memory/InMemory/InMemoryVectorStoreTextSearchTests.cs @@ -1,5 +1,7 @@ // Copyright (c) Microsoft. All rights reserved. +#pragma warning disable CS0618 // ITextSearch is obsolete + using System; using System.Threading.Tasks; using Microsoft.Extensions.AI; diff --git a/dotnet/src/IntegrationTests/Connectors/Memory/Qdrant/QdrantTextSearchTests.cs b/dotnet/src/IntegrationTests/Connectors/Memory/Qdrant/QdrantTextSearchTests.cs index 5a1619138472..5f02d94c4022 100644 --- a/dotnet/src/IntegrationTests/Connectors/Memory/Qdrant/QdrantTextSearchTests.cs +++ b/dotnet/src/IntegrationTests/Connectors/Memory/Qdrant/QdrantTextSearchTests.cs @@ -1,5 +1,7 @@ // Copyright (c) Microsoft. All rights reserved. +#pragma warning disable CS0618 // ITextSearch is obsolete + using System; using System.Threading.Tasks; using Microsoft.SemanticKernel.Connectors.Qdrant; diff --git a/dotnet/src/IntegrationTests/Data/BaseTextSearchTests.cs b/dotnet/src/IntegrationTests/Data/BaseTextSearchTests.cs index 5e7716bcfb3e..3e598f6d546b 100644 --- a/dotnet/src/IntegrationTests/Data/BaseTextSearchTests.cs +++ b/dotnet/src/IntegrationTests/Data/BaseTextSearchTests.cs @@ -1,5 +1,7 @@ // Copyright (c) Microsoft. All rights reserved. +#pragma warning disable CS0618 // Type or member is obsolete - Testing legacy non-generic ITextSearch interface + using System; using System.Collections.Generic; using System.Linq; diff --git a/dotnet/src/IntegrationTests/Plugins/Web/Bing/BingTextSearchTests.cs b/dotnet/src/IntegrationTests/Plugins/Web/Bing/BingTextSearchTests.cs index 34550d130459..bc418182682b 100644 --- a/dotnet/src/IntegrationTests/Plugins/Web/Bing/BingTextSearchTests.cs +++ b/dotnet/src/IntegrationTests/Plugins/Web/Bing/BingTextSearchTests.cs @@ -1,5 +1,7 @@ // Copyright (c) Microsoft. All rights reserved. +#pragma warning disable CS0618 // ITextSearch is obsolete + using System.Threading.Tasks; using Microsoft.Extensions.Configuration; using Microsoft.SemanticKernel.Data; diff --git a/dotnet/src/IntegrationTests/Plugins/Web/Google/GoogleTextSearchTests.cs b/dotnet/src/IntegrationTests/Plugins/Web/Google/GoogleTextSearchTests.cs index 73244ce75d8b..1bf0ba48a232 100644 --- a/dotnet/src/IntegrationTests/Plugins/Web/Google/GoogleTextSearchTests.cs +++ b/dotnet/src/IntegrationTests/Plugins/Web/Google/GoogleTextSearchTests.cs @@ -1,5 +1,7 @@ // Copyright (c) Microsoft. All rights reserved. +#pragma warning disable CS0618 // ITextSearch is obsolete + using System.Threading.Tasks; using Microsoft.Extensions.Configuration; using Microsoft.SemanticKernel.Data; diff --git a/dotnet/src/IntegrationTests/Plugins/Web/Tavily/TavilyTextSearchTests.cs b/dotnet/src/IntegrationTests/Plugins/Web/Tavily/TavilyTextSearchTests.cs index ffc0e066b8d4..77529b8fe1c5 100644 --- a/dotnet/src/IntegrationTests/Plugins/Web/Tavily/TavilyTextSearchTests.cs +++ b/dotnet/src/IntegrationTests/Plugins/Web/Tavily/TavilyTextSearchTests.cs @@ -1,5 +1,7 @@ // Copyright (c) Microsoft. All rights reserved. +#pragma warning disable CS0618 // ITextSearch is obsolete + using System.Threading.Tasks; using Microsoft.Extensions.Configuration; using Microsoft.SemanticKernel.Data; diff --git a/dotnet/src/Plugins/Plugins.UnitTests/Web/Bing/BingTextSearchTests.cs b/dotnet/src/Plugins/Plugins.UnitTests/Web/Bing/BingTextSearchTests.cs index a6172e334314..804f92def433 100644 --- a/dotnet/src/Plugins/Plugins.UnitTests/Web/Bing/BingTextSearchTests.cs +++ b/dotnet/src/Plugins/Plugins.UnitTests/Web/Bing/BingTextSearchTests.cs @@ -1,5 +1,8 @@ // Copyright (c) Microsoft. All rights reserved. +#pragma warning disable CS0618 // ITextSearch is obsolete +#pragma warning disable CS8602 // Dereference of a possibly null reference - Test LINQ expressions access BingWebPage properties guaranteed non-null in test context + using System; using System.IO; using System.Linq; @@ -213,7 +216,7 @@ public async Task BuildsCorrectUriForEqualityFilterAsync(string paramName, objec var requestUris = this._messageHandlerStub.RequestUris; Assert.Single(requestUris); Assert.NotNull(requestUris[0]); - Assert.Equal(requestLink, requestUris[0]!.AbsoluteUri); + Assert.Equal(requestLink, requestUris[0].AbsoluteUri); } [Fact] @@ -231,6 +234,705 @@ public async Task DoesNotBuildsUriForInvalidQueryParameterAsync() Assert.Equal("Unknown equality filter clause field name 'fooBar', must be one of answerCount,cc,freshness,mkt,promote,responseFilter,safeSearch,setLang,textDecorations,textFormat,contains,ext,filetype,inanchor,inbody,intitle,ip,language,loc,location,prefer,site,feed,hasfeed,url (Parameter 'searchOptions')", e.Message); } + #region Generic ITextSearch Interface Tests + + [Fact] + public async Task GenericSearchAsyncWithLanguageEqualityFilterProducesCorrectBingQueryAsync() + { + // Arrange + this._messageHandlerStub.AddJsonResponse(File.ReadAllText(WhatIsTheSKResponseJson)); + ITextSearch textSearch = new BingTextSearch(apiKey: "ApiKey", options: new() { HttpClient = this._httpClient }); + + // Act + var searchOptions = new TextSearchOptions + { + Top = 4, + Skip = 0, + Filter = page => page.Language == "en" + }; + KernelSearchResults result = await textSearch.SearchAsync("What is the Semantic Kernel?", searchOptions); + + // Assert - Verify LINQ expression converted to Bing's language: operator + var requestUris = this._messageHandlerStub.RequestUris; + Assert.Single(requestUris); + Assert.NotNull(requestUris[0]); + Assert.Contains("language%3Aen", requestUris[0].AbsoluteUri); + Assert.Contains("count=4", requestUris[0].AbsoluteUri); + Assert.Contains("offset=0", requestUris[0].AbsoluteUri); + } + + [Fact] + public async Task GenericSearchAsyncWithLanguageInequalityFilterProducesCorrectBingQueryAsync() + { + // Arrange + this._messageHandlerStub.AddJsonResponse(File.ReadAllText(WhatIsTheSKResponseJson)); + ITextSearch textSearch = new BingTextSearch(apiKey: "ApiKey", options: new() { HttpClient = this._httpClient }); + + // Act + var searchOptions = new TextSearchOptions + { + Top = 4, + Skip = 0, + Filter = page => page.Language != "fr" + }; + KernelSearchResults result = await textSearch.SearchAsync("What is the Semantic Kernel?", searchOptions); + + // Assert - Verify LINQ inequality expression converted to Bing's negation syntax (-language:fr) + var requestUris = this._messageHandlerStub.RequestUris; + Assert.Single(requestUris); + Assert.NotNull(requestUris[0]); + Assert.Contains("-language%3Afr", requestUris[0].AbsoluteUri); + } + + [Fact] + public async Task GenericSearchAsyncWithContainsFilterProducesCorrectBingQueryAsync() + { + // Arrange + this._messageHandlerStub.AddJsonResponse(File.ReadAllText(WhatIsTheSKResponseJson)); + ITextSearch textSearch = new BingTextSearch(apiKey: "ApiKey", options: new() { HttpClient = this._httpClient }); + + // Act + var searchOptions = new TextSearchOptions + { + Top = 4, + Skip = 0, + Filter = page => page.Name!.Contains("Microsoft") + }; + KernelSearchResults result = await textSearch.SearchAsync("What is the Semantic Kernel?", searchOptions); + + // Assert - Verify LINQ Contains() converted to Bing's intitle: operator + var requestUris = this._messageHandlerStub.RequestUris; + Assert.Single(requestUris); + Assert.NotNull(requestUris[0]); + Assert.Contains("intitle%3AMicrosoft", requestUris[0].AbsoluteUri); + } + + [Fact] + public async Task GenericSearchAsyncWithComplexAndFilterProducesCorrectBingQueryAsync() + { + // Arrange + this._messageHandlerStub.AddJsonResponse(File.ReadAllText(WhatIsTheSKResponseJson)); + ITextSearch textSearch = new BingTextSearch(apiKey: "ApiKey", options: new() { HttpClient = this._httpClient }); + + // Act + var searchOptions = new TextSearchOptions + { + Top = 4, + Skip = 0, + Filter = page => page.Language == "en" && page.Name!.Contains("AI") + }; + KernelSearchResults result = await textSearch.SearchAsync("What is the Semantic Kernel?", searchOptions); + + // Assert - Verify LINQ AND expression produces both Bing operators + var requestUris = this._messageHandlerStub.RequestUris; + Assert.Single(requestUris); + Assert.NotNull(requestUris[0]); + Assert.Contains("language%3Aen", requestUris[0].AbsoluteUri); + Assert.Contains("intitle%3AAI", requestUris[0].AbsoluteUri); + } + + [Fact] + public async Task GenericGetTextSearchResultsAsyncWithUrlFilterProducesCorrectBingQueryAsync() + { + // Arrange + this._messageHandlerStub.AddJsonResponse(File.ReadAllText(WhatIsTheSKResponseJson)); + ITextSearch textSearch = new BingTextSearch(apiKey: "ApiKey", options: new() { HttpClient = this._httpClient }); + + // Act + var searchOptions = new TextSearchOptions + { + Top = 4, + Skip = 0, + Filter = page => page.Url!.Contains("microsoft.com") + }; + KernelSearchResults result = await textSearch.GetTextSearchResultsAsync("What is the Semantic Kernel?", searchOptions); + + // Assert - Verify LINQ Url.Contains() converted to Bing's url: operator + var requestUris = this._messageHandlerStub.RequestUris; + Assert.Single(requestUris); + Assert.NotNull(requestUris[0]); + Assert.Contains("url%3Amicrosoft.com", requestUris[0].AbsoluteUri); + + // Also verify result structure + Assert.NotNull(result); + Assert.NotNull(result.Results); + } + + [Fact] + public async Task GenericGetSearchResultsAsyncWithSnippetContainsFilterProducesCorrectBingQueryAsync() + { + // Arrange + this._messageHandlerStub.AddJsonResponse(File.ReadAllText(WhatIsTheSKResponseJson)); + ITextSearch textSearch = new BingTextSearch(apiKey: "ApiKey", options: new() { HttpClient = this._httpClient }); + + // Act + var searchOptions = new TextSearchOptions + { + Top = 4, + Skip = 0, + Filter = page => page.Snippet!.Contains("semantic") + }; + KernelSearchResults result = await textSearch.GetSearchResultsAsync("What is the Semantic Kernel?", searchOptions); + + // Assert - Verify LINQ Snippet.Contains() converted to Bing's inbody: operator + var requestUris = this._messageHandlerStub.RequestUris; + Assert.Single(requestUris); + Assert.NotNull(requestUris[0]); + Assert.Contains("inbody%3Asemantic", requestUris[0].AbsoluteUri); + + // Verify result structure + Assert.NotNull(result); + Assert.NotNull(result.Results); + } + + [Fact] + public async Task GenericSearchAsyncWithDisplayUrlEqualityFilterProducesCorrectBingQueryAsync() + { + // Arrange + this._messageHandlerStub.AddJsonResponse(File.ReadAllText(SiteFilterDevBlogsResponseJson)); + ITextSearch textSearch = new BingTextSearch(apiKey: "ApiKey", options: new() { HttpClient = this._httpClient }); + + // Act + var searchOptions = new TextSearchOptions + { + Top = 4, + Skip = 0, + Filter = page => page.DisplayUrl == "devblogs.microsoft.com" + }; + KernelSearchResults result = await textSearch.SearchAsync("What is the Semantic Kernel?", searchOptions); + + // Assert - Verify LINQ DisplayUrl equality converted to Bing's site: operator + var requestUris = this._messageHandlerStub.RequestUris; + Assert.Single(requestUris); + Assert.NotNull(requestUris[0]); + Assert.Contains("site%3Adevblogs.microsoft.com", requestUris[0].AbsoluteUri); + } + + [Fact] + public async Task GenericSearchAsyncWithMultipleAndConditionsProducesCorrectBingQueryAsync() + { + // Arrange + this._messageHandlerStub.AddJsonResponse(File.ReadAllText(WhatIsTheSKResponseJson)); + ITextSearch textSearch = new BingTextSearch(apiKey: "ApiKey", options: new() { HttpClient = this._httpClient }); + + // Act + var searchOptions = new TextSearchOptions + { + Top = 4, + Skip = 0, + Filter = page => page.Language == "en" && page.DisplayUrl!.Contains("microsoft.com") && page.Name!.Contains("Semantic") + }; + KernelSearchResults result = await textSearch.SearchAsync("What is the Semantic Kernel?", searchOptions); + + // Assert - Verify all LINQ conditions converted correctly + var requestUris = this._messageHandlerStub.RequestUris; + Assert.Single(requestUris); + Assert.NotNull(requestUris[0]); + string uri = requestUris[0].AbsoluteUri; + Assert.Contains("language%3Aen", uri); + Assert.Contains("site%3Amicrosoft.com", uri); // DisplayUrl.Contains() ? site: operator + Assert.Contains("intitle%3ASemantic", uri); + } + + [Fact] + public async Task GenericSearchAsyncWithNoFilterReturnsResultsSuccessfullyAsync() + { + // Arrange + this._messageHandlerStub.AddJsonResponse(File.ReadAllText(WhatIsTheSKResponseJson)); + ITextSearch textSearch = new BingTextSearch(apiKey: "ApiKey", options: new() { HttpClient = this._httpClient }); + + // Act - No filter specified + var searchOptions = new TextSearchOptions + { + Top = 10, + Skip = 0 + }; + KernelSearchResults result = await textSearch.SearchAsync("What is the Semantic Kernel?", searchOptions); + + // Assert - Verify basic query without filter operators + var requestUris = this._messageHandlerStub.RequestUris; + Assert.Single(requestUris); + Assert.NotNull(requestUris[0]); + Assert.DoesNotContain("language%3A", requestUris[0].AbsoluteUri); + Assert.DoesNotContain("intitle%3A", requestUris[0].AbsoluteUri); + + // Verify results + Assert.NotNull(result); + Assert.NotNull(result.Results); + var resultList = await result.Results.ToListAsync(); + Assert.Equal(10, resultList.Count); + } + + [Fact] + public async Task GenericSearchAsyncWithIsFamilyFriendlyFilterProducesCorrectBingQueryAsync() + { + // Arrange + this._messageHandlerStub.AddJsonResponse(File.ReadAllText(WhatIsTheSKResponseJson)); + ITextSearch textSearch = new BingTextSearch(apiKey: "ApiKey", options: new() { HttpClient = this._httpClient }); + + // Act + var searchOptions = new TextSearchOptions + { + Top = 4, + Skip = 0, + Filter = page => page.IsFamilyFriendly == true + }; + KernelSearchResults result = await textSearch.SearchAsync("What is the Semantic Kernel?", searchOptions); + + // Assert - Verify LINQ IsFamilyFriendly equality converted to Bing's safeSearch query parameter + var requestUris = this._messageHandlerStub.RequestUris; + Assert.Single(requestUris); + Assert.NotNull(requestUris[0]); + // safeSearch is a query parameter, not an advanced search operator + Assert.Contains("safeSearch=true", requestUris[0].AbsoluteUri); + } + + [Fact] + public async Task GenericSearchAsyncWithIsFamilyFriendlyFalseFilterProducesCorrectBingQueryAsync() + { + // Arrange + this._messageHandlerStub.AddJsonResponse(File.ReadAllText(WhatIsTheSKResponseJson)); + ITextSearch textSearch = new BingTextSearch(apiKey: "ApiKey", options: new() { HttpClient = this._httpClient }); + + // Act + var searchOptions = new TextSearchOptions + { + Top = 4, + Skip = 0, + Filter = page => page.IsFamilyFriendly == false + }; + KernelSearchResults result = await textSearch.SearchAsync("What is the Semantic Kernel?", searchOptions); + + // Assert - Verify false value converted correctly + var requestUris = this._messageHandlerStub.RequestUris; + Assert.Single(requestUris); + Assert.NotNull(requestUris[0]); + Assert.Contains("safeSearch=false", requestUris[0].AbsoluteUri); + } + + [Fact] + public async Task GenericSearchAsyncWithMultipleContainsConditionsProducesCorrectBingQueryAsync() + { + // Arrange + this._messageHandlerStub.AddJsonResponse(File.ReadAllText(WhatIsTheSKResponseJson)); + ITextSearch textSearch = new BingTextSearch(apiKey: "ApiKey", options: new() { HttpClient = this._httpClient }); + + // Act - Multiple Contains operations on different properties + var searchOptions = new TextSearchOptions + { + Top = 4, + Skip = 0, + Filter = page => page.Name!.Contains("Semantic") && page.Snippet!.Contains("kernel") && page.Url!.Contains("microsoft.com") + }; + KernelSearchResults result = await textSearch.SearchAsync("AI", searchOptions); + + // Assert - Verify all Contains operations translated correctly + var requestUris = this._messageHandlerStub.RequestUris; + Assert.Single(requestUris); + Assert.NotNull(requestUris[0]); + string uri = requestUris[0].AbsoluteUri; + Assert.Contains("intitle%3ASemantic", uri); // Name.Contains() ? intitle: + Assert.Contains("inbody%3Akernel", uri); // Snippet.Contains() ? inbody: + Assert.Contains("url%3Amicrosoft.com", uri); // Url.Contains() ? url: + } + + [Fact] + public async Task GenericSearchAsyncWithMixedEqualityAndContainsProducesCorrectBingQueryAsync() + { + // Arrange + this._messageHandlerStub.AddJsonResponse(File.ReadAllText(WhatIsTheSKResponseJson)); + ITextSearch textSearch = new BingTextSearch(apiKey: "ApiKey", options: new() { HttpClient = this._httpClient }); + + // Act - Mix equality and Contains operations + var searchOptions = new TextSearchOptions + { + Top = 4, + Skip = 0, + Filter = page => page.Language == "en" && + page.IsFamilyFriendly == true && + page.Name!.Contains("Azure") && + page.DisplayUrl!.Contains("microsoft.com") + }; + KernelSearchResults result = await textSearch.SearchAsync("cloud", searchOptions); + + // Assert - Verify mixed operators all present + var requestUris = this._messageHandlerStub.RequestUris; + Assert.Single(requestUris); + Assert.NotNull(requestUris[0]); + string uri = requestUris[0].AbsoluteUri; + Assert.Contains("language%3Aen", uri); + Assert.Contains("safeSearch=true", uri); + Assert.Contains("intitle%3AAzure", uri); + Assert.Contains("site%3Amicrosoft.com", uri); + } + + [Fact] + public async Task GenericSearchAsyncWithInequalityAndEqualityProducesCorrectBingQueryAsync() + { + // Arrange + this._messageHandlerStub.AddJsonResponse(File.ReadAllText(WhatIsTheSKResponseJson)); + ITextSearch textSearch = new BingTextSearch(apiKey: "ApiKey", options: new() { HttpClient = this._httpClient }); + + // Act - Combine inequality (negation) with positive equality + var searchOptions = new TextSearchOptions + { + Top = 4, + Skip = 0, + Filter = page => page.Language != "fr" && page.DisplayUrl == "docs.microsoft.com" + }; + KernelSearchResults result = await textSearch.SearchAsync("documentation", searchOptions); + + // Assert - Verify negation and positive condition both present + var requestUris = this._messageHandlerStub.RequestUris; + Assert.Single(requestUris); + Assert.NotNull(requestUris[0]); + string uri = requestUris[0].AbsoluteUri; + Assert.Contains("-language%3Afr", uri); // Negation prefix + Assert.Contains("site%3Adocs.microsoft.com", uri); // Positive condition + } + + [Fact] + public async Task GenericSearchAsyncWithUrlAndDisplayUrlBothProducesCorrectOperatorsAsync() + { + // Arrange + this._messageHandlerStub.AddJsonResponse(File.ReadAllText(WhatIsTheSKResponseJson)); + ITextSearch textSearch = new BingTextSearch(apiKey: "ApiKey", options: new() { HttpClient = this._httpClient }); + + // Act - Use both Url and DisplayUrl properties + var searchOptions = new TextSearchOptions + { + Top = 4, + Skip = 0, + Filter = page => page.Url!.Contains("github.com") && page.DisplayUrl!.Contains("microsoft") + }; + KernelSearchResults result = await textSearch.SearchAsync("repository", searchOptions); + + // Assert - Both should map to their respective operators + var requestUris = this._messageHandlerStub.RequestUris; + Assert.Single(requestUris); + Assert.NotNull(requestUris[0]); + string uri = requestUris[0].AbsoluteUri; + Assert.Contains("url%3Agithub.com", uri); // Url.Contains() ? url: + Assert.Contains("site%3Amicrosoft", uri); // DisplayUrl.Contains() ? site: + } + + [Fact] + public async Task GenericSearchAsyncWithComplexFourConditionFilterProducesCorrectBingQueryAsync() + { + // Arrange + this._messageHandlerStub.AddJsonResponse(File.ReadAllText(WhatIsTheSKResponseJson)); + ITextSearch textSearch = new BingTextSearch(apiKey: "ApiKey", options: new() { HttpClient = this._httpClient }); + + // Act - Complex filter with 4 AND conditions testing multiple operator types + var searchOptions = new TextSearchOptions + { + Top = 4, + Skip = 0, + Filter = page => page.Language == "en" && + page.Language != "fr" && // This should be ignored (contradiction) + page.Name!.Contains("Tutorial") && + page.Snippet!.Contains("beginner") + }; + KernelSearchResults result = await textSearch.SearchAsync("learn", searchOptions); + + // Assert - Verify all conditions present (including contradictory ones) + var requestUris = this._messageHandlerStub.RequestUris; + Assert.Single(requestUris); + Assert.NotNull(requestUris[0]); + string uri = requestUris[0].AbsoluteUri; + Assert.Contains("language%3Aen", uri); + Assert.Contains("-language%3Afr", uri); // Both positive and negative language (contradictory but valid) + Assert.Contains("intitle%3ATutorial", uri); + Assert.Contains("inbody%3Abeginner", uri); + } + + [Fact] + public async Task GenericSearchAsyncWithSpecialCharactersInContainsValueProducesCorrectEncodingAsync() + { + // Arrange + this._messageHandlerStub.AddJsonResponse(File.ReadAllText(WhatIsTheSKResponseJson)); + ITextSearch textSearch = new BingTextSearch(apiKey: "ApiKey", options: new() { HttpClient = this._httpClient }); + + // Act - Contains with special characters that need URL encoding + var searchOptions = new TextSearchOptions + { + Top = 4, + Skip = 0, + Filter = page => page.Name!.Contains("C# & .NET") + }; + KernelSearchResults result = await textSearch.SearchAsync("programming", searchOptions); + + // Assert - Verify special characters are URL-encoded properly + var requestUris = this._messageHandlerStub.RequestUris; + Assert.Single(requestUris); + Assert.NotNull(requestUris[0]); + string uri = requestUris[0].AbsoluteUri; + // Should contain URL-encoded version of "C# & .NET" + Assert.Contains("intitle%3A", uri); + // Verify the query was constructed (exact encoding may vary) + Assert.True(uri.Contains("intitle"), "Should contain intitle operator"); + } + + [Fact] + public async Task GenericSearchAsyncWithEmptyFilterProducesBaseQueryAsync() + { + // Arrange + this._messageHandlerStub.AddJsonResponse(File.ReadAllText(WhatIsTheSKResponseJson)); + ITextSearch textSearch = new BingTextSearch(apiKey: "ApiKey", options: new() { HttpClient = this._httpClient }); + + // Act - Explicit null filter (should be treated like no filter) + var searchOptions = new TextSearchOptions + { + Top = 5, + Skip = 0, + Filter = null + }; + KernelSearchResults result = await textSearch.SearchAsync("test query", searchOptions); + + // Assert - Should produce basic query without filter operators + var requestUris = this._messageHandlerStub.RequestUris; + Assert.Single(requestUris); + Assert.NotNull(requestUris[0]); + string uri = requestUris[0].AbsoluteUri; + Assert.Contains("test", uri); // Query should be present (URL-encoded) + Assert.Contains("count=5", uri); + Assert.DoesNotContain("intitle%3A", uri); + Assert.DoesNotContain("language%3A", uri); + } + + [Fact] + public async Task GenericSearchAsyncWithOnlyInequalityFilterProducesNegationAsync() + { + // Arrange + this._messageHandlerStub.AddJsonResponse(File.ReadAllText(WhatIsTheSKResponseJson)); + ITextSearch textSearch = new BingTextSearch(apiKey: "ApiKey", options: new() { HttpClient = this._httpClient }); + + // Act - Only inequality (pure negation) + var searchOptions = new TextSearchOptions + { + Top = 4, + Skip = 0, + Filter = page => page.Language != "es" + }; + KernelSearchResults result = await textSearch.SearchAsync("content", searchOptions); + + // Assert - Verify negation operator present + var requestUris = this._messageHandlerStub.RequestUris; + Assert.Single(requestUris); + Assert.NotNull(requestUris[0]); + Assert.Contains("-language%3Aes", requestUris[0].AbsoluteUri); + } + + [Fact] + public async Task GenericSearchAsyncWithIsFamilyFriendlyInequalityProducesNegatedSafeSearchAsync() + { + // Arrange + this._messageHandlerStub.AddJsonResponse(File.ReadAllText(WhatIsTheSKResponseJson)); + ITextSearch textSearch = new BingTextSearch(apiKey: "ApiKey", options: new() { HttpClient = this._httpClient }); + + // Act - IsFamilyFriendly with inequality (converts to negated value) + var searchOptions = new TextSearchOptions + { + Top = 4, + Skip = 0, + Filter = page => page.IsFamilyFriendly != true + }; + KernelSearchResults result = await textSearch.SearchAsync("content", searchOptions); + + // Assert - Verify negated boolean converted properly (note: safeSearch is a query parameter, not an advanced search keyword) + // Query parameters don't support negation prefix like advanced search keywords, so false != true becomes -true value + var requestUris = this._messageHandlerStub.RequestUris; + Assert.Single(requestUris); + Assert.NotNull(requestUris[0]); + string uri = requestUris[0].AbsoluteUri; + // The actual behavior: != true gets processed as negation marker, resulting in safeSearch=-true (treated as invalid/ignored) + // This test documents current behavior - inequality on boolean query params has limitations + Assert.Contains("safeSearch", uri); + } + + [Fact] + public async Task GenericSearchAsyncWithContainsOnNameAndUrlProducesDistinctOperatorsAsync() + { + // Arrange + this._messageHandlerStub.AddJsonResponse(File.ReadAllText(WhatIsTheSKResponseJson)); + ITextSearch textSearch = new BingTextSearch(apiKey: "ApiKey", options: new() { HttpClient = this._httpClient }); + + // Act - Same search term in different properties should use different operators + var searchOptions = new TextSearchOptions + { + Top = 4, + Skip = 0, + Filter = page => page.Name!.Contains("docs") && page.Url!.Contains("docs") + }; + KernelSearchResults result = await textSearch.SearchAsync("documentation", searchOptions); + + // Assert - Verify both operators present despite same search term + var requestUris = this._messageHandlerStub.RequestUris; + Assert.Single(requestUris); + Assert.NotNull(requestUris[0]); + string uri = requestUris[0].AbsoluteUri; + Assert.Contains("intitle%3Adocs", uri); // Name ? intitle: + Assert.Contains("url%3Adocs", uri); // Url ? url: + // Verify both operators are present (not deduplicated) + Assert.Equal(2, System.Text.RegularExpressions.Regex.Matches(uri, "docs").Count); + } + + [Fact] + public async Task GenericSearchAsyncFilterTranslationPreservesResultStructureAsync() + { + // Arrange + this._messageHandlerStub.AddJsonResponse(File.ReadAllText(WhatIsTheSKResponseJson)); + ITextSearch textSearch = new BingTextSearch(apiKey: "ApiKey", options: new() { HttpClient = this._httpClient }); + + // Act - Complex filter to ensure result structure not affected by filtering + var searchOptions = new TextSearchOptions + { + Top = 10, + Skip = 0, + Filter = page => page.Language == "en" && page.Name!.Contains("Kernel") + }; + KernelSearchResults result = await textSearch.SearchAsync("AI", searchOptions); + + // Assert - Verify results are properly structured + Assert.NotNull(result); + Assert.NotNull(result.Results); + var resultList = await result.Results.ToListAsync(); + Assert.Equal(10, resultList.Count); + foreach (var item in resultList) + { + Assert.NotNull(item); + Assert.NotEmpty(item); // Each result should be non-empty string + } + } + + [Fact] + public async Task GenericGetTextSearchResultsAsyncFilterTranslationPreservesMetadataAsync() + { + // Arrange + this._messageHandlerStub.AddJsonResponse(File.ReadAllText(WhatIsTheSKResponseJson)); + ITextSearch textSearch = new BingTextSearch(apiKey: "ApiKey", options: new() { HttpClient = this._httpClient }); + + // Act - Use GetTextSearchResultsAsync with filter to verify metadata preservation + var searchOptions = new TextSearchOptions + { + Top = 10, + Skip = 0, + Filter = page => page.Snippet!.Contains("semantic") + }; + KernelSearchResults result = await textSearch.GetTextSearchResultsAsync("Kernel", searchOptions); + + // Assert - Verify TextSearchResult structure with Name, Value, Link + Assert.NotNull(result); + Assert.NotNull(result.Results); + var resultList = await result.Results.ToListAsync(); + Assert.Equal(10, resultList.Count); + foreach (var textSearchResult in resultList) + { + Assert.NotNull(textSearchResult); + Assert.NotNull(textSearchResult.Name); + Assert.NotNull(textSearchResult.Value); + Assert.NotNull(textSearchResult.Link); + Assert.NotEmpty(textSearchResult.Name); + Assert.NotEmpty(textSearchResult.Value); + Assert.NotEmpty(textSearchResult.Link); + } + } + + [Fact] + public async Task GenericGetSearchResultsAsyncFilterTranslationPreservesBingWebPageStructureAsync() + { + // Arrange + this._messageHandlerStub.AddJsonResponse(File.ReadAllText(WhatIsTheSKResponseJson)); + ITextSearch textSearch = new BingTextSearch(apiKey: "ApiKey", options: new() { HttpClient = this._httpClient }); + + // Act - Use GetSearchResultsAsync with filter to get raw BingWebPage objects + var searchOptions = new TextSearchOptions + { + Top = 10, + Skip = 0, + Filter = page => page.Language == "en" && page.DisplayUrl!.Contains("microsoft") + }; + KernelSearchResults result = await textSearch.GetSearchResultsAsync("technology", searchOptions); + + // Assert - Verify BingWebPage objects have all expected properties + Assert.NotNull(result); + Assert.NotNull(result.Results); + var resultList = await result.Results.ToListAsync(); + Assert.Equal(10, resultList.Count); + foreach (var page in resultList) + { + Assert.NotNull(page); + // Verify key properties are populated - now strongly typed, no cast needed! + Assert.NotNull(page.Name); + Assert.NotNull(page.Url); + Assert.NotNull(page.Snippet); + // DisplayUrl might be null in some cases, so don't assert NotNull + } + } + + [Fact] + public async Task CollectionContainsFilterThrowsNotSupportedExceptionAsync() + { + // Arrange - Tests both Enumerable.Contains (C# 13-) and MemoryExtensions.Contains (C# 14+) + // The same code array.Contains() resolves differently based on C# language version: + // - C# 13 and earlier: Enumerable.Contains (LINQ extension method) + // - C# 14 and later: MemoryExtensions.Contains (span-based optimization due to "first-class spans") + // Our implementation handles both identically since Bing API doesn't support OR logic for either + this._messageHandlerStub.AddJsonResponse(File.ReadAllText(WhatIsTheSKResponseJson)); + ITextSearch textSearch = new BingTextSearch(apiKey: "ApiKey", options: new() { HttpClient = this._httpClient }); + string[] languages = ["en", "fr"]; + + // Act & Assert - Verify that collection Contains pattern throws clear exception + var searchOptions = new TextSearchOptions + { + Top = 5, + Skip = 0, + Filter = page => languages.Contains(page.Language!) // Enumerable.Contains (C# 13-) or MemoryExtensions.Contains (C# 14+) + }; + + var exception = await Assert.ThrowsAsync(async () => + { + await textSearch.SearchAsync("test", searchOptions); + }); + + // Assert - Verify error message explains the limitation clearly + Assert.Contains("Collection Contains filters", exception.Message); + Assert.Contains("not supported by Bing Search API", exception.Message); + Assert.Contains("OR logic", exception.Message); + } + + [Fact] + public async Task StringContainsStillWorksWithLINQFiltersAsync() + { + // Arrange - Verify that String.Contains (instance method) still works + // String.Contains is NOT affected by C# 14 "first-class spans" - only arrays are + this._messageHandlerStub.AddJsonResponse(File.ReadAllText(WhatIsTheSKResponseJson)); + ITextSearch textSearch = new BingTextSearch(apiKey: "ApiKey", options: new() { HttpClient = this._httpClient }); + + // Act - String.Contains should continue to work + var searchOptions = new TextSearchOptions + { + Top = 5, + Skip = 0, + Filter = page => page.Name.Contains("Semantic") // String.Contains - instance method + }; + + KernelSearchResults result = await textSearch.SearchAsync("test", searchOptions); + + // Assert - Should succeed without exception + Assert.NotNull(result); + Assert.NotNull(result.Results); + var resultsList = await result.Results.ToListAsync(); + Assert.NotEmpty(resultsList); + + // Verify the filter was translated correctly to intitle: operator + var requestUris = this._messageHandlerStub.RequestUris; + Assert.Single(requestUris); + Assert.Contains("intitle%3ASemantic", requestUris[0]!.AbsoluteUri); + } + + #endregion + /// public void Dispose() { diff --git a/dotnet/src/Plugins/Plugins.UnitTests/Web/Brave/BraveTextSearchTests.cs b/dotnet/src/Plugins/Plugins.UnitTests/Web/Brave/BraveTextSearchTests.cs index 8a98a3d81a47..84f7a3a478e9 100644 --- a/dotnet/src/Plugins/Plugins.UnitTests/Web/Brave/BraveTextSearchTests.cs +++ b/dotnet/src/Plugins/Plugins.UnitTests/Web/Brave/BraveTextSearchTests.cs @@ -1,5 +1,8 @@ // Copyright (c) Microsoft. All rights reserved. +#pragma warning disable CS0618 // ITextSearch is obsolete +#pragma warning disable CS8602 // Dereference of a possibly null reference - for LINQ expression properties + using System; using System.IO; using System.Linq; @@ -108,7 +111,7 @@ public async Task GetSearchResultsReturnsSuccessfullyAsync() var resultList = await result.Results.ToListAsync(); Assert.NotNull(resultList); Assert.Equal(10, resultList.Count); - foreach (BraveWebResult webPage in resultList) + foreach (BraveWebPage webPage in resultList.Cast()) { Assert.NotNull(webPage.Title); Assert.NotNull(webPage.Description); @@ -193,7 +196,7 @@ public async Task BuildsCorrectUriForEqualityFilterAsync(string paramName, objec // Act TextSearchOptions searchOptions = new() { Top = 5, Skip = 0, Filter = new TextSearchFilter().Equality(paramName, paramValue) }; - KernelSearchResults result = await textSearch.GetSearchResultsAsync("What is the Semantic Kernel?", searchOptions); + var result = await textSearch.GetSearchResultsAsync("What is the Semantic Kernel?", searchOptions); // Assert var requestUris = this._messageHandlerStub.RequestUris; @@ -241,6 +244,151 @@ public void Dispose() GC.SuppressFinalize(this); } + #region Generic ITextSearch Interface Tests + + [Fact] + public async Task LinqSearchAsyncReturnsResultsSuccessfullyAsync() + { + // Arrange + this._messageHandlerStub.AddJsonResponse(File.ReadAllText(WhatIsTheSkResponseJson)); + ITextSearch textSearch = new BraveTextSearch(apiKey: "ApiKey", options: new() { HttpClient = this._httpClient }); + + // Act + var searchOptions = new TextSearchOptions + { + Top = 4, + Skip = 0 + }; + KernelSearchResults result = await textSearch.SearchAsync("What is the Semantic Kernel?", searchOptions); + + // Assert - Verify basic generic interface functionality + Assert.NotNull(result); + Assert.NotNull(result.Results); + var resultList = await result.Results.ToListAsync(); + Assert.NotEmpty(resultList); + + // Verify the request was made correctly + var requestUris = this._messageHandlerStub.RequestUris; + Assert.Single(requestUris); + Assert.NotNull(requestUris[0]); + Assert.Contains("count=4", requestUris[0].AbsoluteUri); + } + + [Fact] + public async Task LinqGetSearchResultsAsyncReturnsResultsSuccessfullyAsync() + { + // Arrange + this._messageHandlerStub.AddJsonResponse(File.ReadAllText(WhatIsTheSkResponseJson)); + ITextSearch textSearch = new BraveTextSearch(apiKey: "ApiKey", options: new() { HttpClient = this._httpClient }); + + // Act + var searchOptions = new TextSearchOptions + { + Top = 3, + Skip = 0 + }; + KernelSearchResults result = await textSearch.GetSearchResultsAsync("What is the Semantic Kernel?", searchOptions); + + // Assert - Verify generic interface returns results + Assert.NotNull(result); + Assert.NotNull(result.Results); + var resultList = await result.Results.ToListAsync(); + Assert.NotEmpty(resultList); + // Results are now strongly typed as BraveWebPage + + // Verify the request was made correctly + var requestUris = this._messageHandlerStub.RequestUris; + Assert.Single(requestUris); + Assert.NotNull(requestUris[0]); + Assert.Contains("count=3", requestUris[0].AbsoluteUri); + } + + [Fact] + public async Task LinqGetTextSearchResultsAsyncReturnsResultsSuccessfullyAsync() + { + // Arrange + this._messageHandlerStub.AddJsonResponse(File.ReadAllText(WhatIsTheSkResponseJson)); + ITextSearch textSearch = new BraveTextSearch(apiKey: "ApiKey", options: new() { HttpClient = this._httpClient }); + + // Act + var searchOptions = new TextSearchOptions + { + Top = 5, + Skip = 0 + }; + KernelSearchResults result = await textSearch.GetTextSearchResultsAsync("What is the Semantic Kernel?", searchOptions); + + // Assert - Verify generic interface returns TextSearchResult objects + Assert.NotNull(result); + Assert.NotNull(result.Results); + var resultList = await result.Results.ToListAsync(); + Assert.NotEmpty(resultList); + Assert.All(resultList, item => Assert.IsType(item)); + + // Verify the request was made correctly + var requestUris = this._messageHandlerStub.RequestUris; + Assert.Single(requestUris); + Assert.NotNull(requestUris[0]); + Assert.Contains("count=5", requestUris[0].AbsoluteUri); + } + + [Fact] + public async Task CollectionContainsFilterThrowsNotSupportedExceptionAsync() + { + // Arrange - Tests both Enumerable.Contains (C# 13-) and MemoryExtensions.Contains (C# 14+) + // The same code array.Contains() resolves differently based on C# language version: + // - C# 13 and earlier: Enumerable.Contains (LINQ extension method) + // - C# 14 and later: MemoryExtensions.Contains (span-based optimization due to "first-class spans") + // Our implementation handles both identically since Brave API has limited query operators + this._messageHandlerStub.AddJsonResponse(File.ReadAllText(WhatIsTheSkResponseJson)); + ITextSearch textSearch = new BraveTextSearch(apiKey: "ApiKey", options: new() { HttpClient = this._httpClient }); + string[] sites = ["microsoft.com", "github.com"]; + + // Act & Assert - Verify that collection Contains pattern throws clear exception + var searchOptions = new TextSearchOptions + { + Top = 5, + Skip = 0, + Filter = page => sites.Contains(page.Url!.ToString()) // Enumerable.Contains (C# 13-) or MemoryExtensions.Contains (C# 14+) + }; + + var exception = await Assert.ThrowsAsync(async () => + { + await textSearch.SearchAsync("test", searchOptions); + }); + + // Assert - Verify error message explains the limitation clearly + Assert.Contains("Collection Contains filters", exception.Message); + Assert.Contains("not supported", exception.Message); + } + + [Fact] + public async Task StringContainsStillWorksWithLINQFiltersAsync() + { + // Arrange - Verify that String.Contains (instance method) still works + // String.Contains is NOT affected by C# 14 "first-class spans" - only arrays are + this._messageHandlerStub.AddJsonResponse(File.ReadAllText(WhatIsTheSkResponseJson)); + ITextSearch textSearch = new BraveTextSearch(apiKey: "ApiKey", options: new() { HttpClient = this._httpClient }); + + // Act - String.Contains should continue to work + var searchOptions = new TextSearchOptions + { + Top = 5, + Skip = 0, + Filter = page => page.Title.Contains("Kernel") // String.Contains - instance method + }; + KernelSearchResults result = await textSearch.SearchAsync("Semantic Kernel tutorial", searchOptions); + + // Assert - Verify String.Contains works correctly + var requestUris = this._messageHandlerStub.RequestUris; + Assert.Single(requestUris); + Assert.NotNull(requestUris[0]); + Assert.Contains("Kernel", requestUris[0].AbsoluteUri); + Assert.Contains("count=5", requestUris[0].AbsoluteUri); + } + + #endregion + #region private private const string WhatIsTheSkResponseJson = "./TestData/brave_what_is_the_semantic_kernel.json"; private const string SiteFilterSkResponseJson = "./TestData/brave_site_filter_what_is_the_semantic_kernel.json"; @@ -271,7 +419,7 @@ public TextSearchResult MapFromResultToTextSearchResult(object result) { if (result is not BraveWebResult webPage) { - throw new ArgumentException("Result must be a BraveWebPage", nameof(result)); + throw new ArgumentException("Result must be a BraveWebResult", nameof(result)); } return new TextSearchResult(webPage.Description?.ToUpperInvariant() ?? string.Empty) diff --git a/dotnet/src/Plugins/Plugins.UnitTests/Web/Google/GoogleTextSearchTests.cs b/dotnet/src/Plugins/Plugins.UnitTests/Web/Google/GoogleTextSearchTests.cs index 1d97ae8ec26b..5eeb12c61c43 100644 --- a/dotnet/src/Plugins/Plugins.UnitTests/Web/Google/GoogleTextSearchTests.cs +++ b/dotnet/src/Plugins/Plugins.UnitTests/Web/Google/GoogleTextSearchTests.cs @@ -1,5 +1,7 @@ // Copyright (c) Microsoft. All rights reserved. +#pragma warning disable CS0618 // ITextSearch is obsolete + using System; using System.IO; using System.Linq; @@ -53,7 +55,7 @@ public async Task SearchReturnsSuccessfullyAsync() searchEngineId: "SearchEngineId"); // Act - KernelSearchResults result = await textSearch.SearchAsync("What is the Semantic Kernel?", new() { Top = 4, Skip = 0 }); + KernelSearchResults result = await textSearch.SearchAsync("What is the Semantic Kernel?", new TextSearchOptions { Top = 4, Skip = 0 }); // Assert Assert.NotNull(result); @@ -79,7 +81,7 @@ public async Task GetTextSearchResultsReturnsSuccessfullyAsync() searchEngineId: "SearchEngineId"); // Act - KernelSearchResults result = await textSearch.GetTextSearchResultsAsync("What is the Semantic Kernel?", new() { Top = 10, Skip = 0 }); + KernelSearchResults result = await textSearch.GetTextSearchResultsAsync("What is the Semantic Kernel?", new TextSearchOptions { Top = 10, Skip = 0 }); // Assert Assert.NotNull(result); @@ -107,7 +109,7 @@ public async Task GetSearchResultsReturnsSuccessfullyAsync() searchEngineId: "SearchEngineId"); // Act - KernelSearchResults results = await textSearch.GetSearchResultsAsync("What is the Semantic Kernel?", new() { Top = 10, Skip = 0 }); + KernelSearchResults results = await textSearch.GetSearchResultsAsync("What is the Semantic Kernel?", new TextSearchOptions { Top = 10, Skip = 0 }); // Assert Assert.NotNull(results); @@ -138,7 +140,7 @@ public async Task SearchWithCustomStringMapperReturnsSuccessfullyAsync() options: new() { StringMapper = new TestTextSearchStringMapper() }); // Act - KernelSearchResults result = await textSearch.SearchAsync("What is the Semantic Kernel?", new() { Top = 4, Skip = 0 }); + KernelSearchResults result = await textSearch.SearchAsync("What is the Semantic Kernel?", new TextSearchOptions { Top = 4, Skip = 0 }); // Assert Assert.NotNull(result); @@ -167,7 +169,7 @@ public async Task GetTextSearchResultsWithCustomResultMapperReturnsSuccessfullyA options: new() { ResultMapper = new TestTextSearchResultMapper() }); // Act - KernelSearchResults result = await textSearch.GetTextSearchResultsAsync("What is the Semantic Kernel?", new() { Top = 4, Skip = 0 }); + KernelSearchResults result = await textSearch.GetTextSearchResultsAsync("What is the Semantic Kernel?", new TextSearchOptions { Top = 4, Skip = 0 }); // Assert Assert.NotNull(result); @@ -232,9 +234,505 @@ public async Task DoesNotBuildsUriForInvalidQueryParameterAsync() // Act && Assert var e = await Assert.ThrowsAsync(async () => await textSearch.GetSearchResultsAsync("What is the Semantic Kernel?", searchOptions)); - Assert.Equal("Unknown equality filter clause field name 'fooBar', must be one of cr,dateRestrict,exactTerms,excludeTerms,filter,gl,hl,linkSite,lr,orTerms,rights,siteSearch (Parameter 'searchOptions')", e.Message); + Assert.Equal("Unknown equality filter clause field name 'fooBar', must be one of cr,dateRestrict,exactTerms,excludeTerms,fileType,filter,gl,hl,linkSite,lr,orTerms,rights,siteSearch (Parameter 'searchOptions')", e.Message); + } + + [Fact] + public async Task GenericSearchAsyncReturnsSuccessfullyAsync() + { + // Arrange + this._messageHandlerStub.AddJsonResponse(File.ReadAllText(WhatIsTheSKResponseJson)); + + // Create an ITextSearch instance using Google search + using var textSearch = new GoogleTextSearch( + initializer: new() { ApiKey = "ApiKey", HttpClientFactory = this._clientFactory }, + searchEngineId: "SearchEngineId"); + + // Act - Use generic interface with GoogleWebPage + KernelSearchResults result = await textSearch.SearchAsync("What is the Semantic Kernel?", new TextSearchOptions { Top = 4, Skip = 0 }); + + // Assert + Assert.NotNull(result); + Assert.NotNull(result.Results); + var resultList = await result.Results.ToListAsync(); + Assert.NotNull(resultList); + Assert.Equal(4, resultList.Count); + foreach (var stringResult in resultList) + { + Assert.NotEmpty(stringResult); + } + } + + [Fact] + public async Task GenericGetTextSearchResultsReturnsSuccessfullyAsync() + { + // Arrange + this._messageHandlerStub.AddJsonResponse(File.ReadAllText(WhatIsTheSKResponseJson)); + + // Create an ITextSearch instance using Google search + using var textSearch = new GoogleTextSearch( + initializer: new() { ApiKey = "ApiKey", HttpClientFactory = this._clientFactory }, + searchEngineId: "SearchEngineId"); + + // Act - Use generic interface with GoogleWebPage + KernelSearchResults result = await textSearch.GetTextSearchResultsAsync("What is the Semantic Kernel?", new TextSearchOptions { Top = 10, Skip = 0 }); + + // Assert + Assert.NotNull(result); + Assert.NotNull(result.Results); + var resultList = await result.Results.ToListAsync(); + Assert.NotNull(resultList); + Assert.Equal(4, resultList.Count); + foreach (var textSearchResult in resultList) + { + Assert.NotNull(textSearchResult.Name); + Assert.NotNull(textSearchResult.Value); + Assert.NotNull(textSearchResult.Link); + } + } + + [Fact] + public async Task GenericGetSearchResultsReturnsSuccessfullyAsync() + { + // Arrange + this._messageHandlerStub.AddJsonResponse(File.ReadAllText(WhatIsTheSKResponseJson)); + + // Create an ITextSearch instance using Google search + using var textSearch = new GoogleTextSearch( + initializer: new() { ApiKey = "ApiKey", HttpClientFactory = this._clientFactory }, + searchEngineId: "SearchEngineId"); + + // Act - Use generic interface with GoogleWebPage + KernelSearchResults results = await textSearch.GetSearchResultsAsync("What is the Semantic Kernel?", new TextSearchOptions { Top = 10, Skip = 0 }); + + // Assert + Assert.NotNull(results); + Assert.NotNull(results.Results); + var resultList = await results.Results.ToListAsync(); + Assert.NotNull(resultList); + Assert.Equal(4, resultList.Count); + foreach (GoogleWebPage result in resultList) + { + Assert.NotNull(result.Title); + Assert.NotNull(result.Snippet); + Assert.NotNull(result.Link); + Assert.NotNull(result.DisplayLink); + } + } + + [Fact] + public async Task GenericSearchWithContainsFilterReturnsSuccessfullyAsync() + { + // Arrange + this._messageHandlerStub.AddJsonResponse(File.ReadAllText(WhatIsTheSKResponseJson)); + + using var textSearch = new GoogleTextSearch( + initializer: new() { ApiKey = "ApiKey", HttpClientFactory = this._clientFactory }, + searchEngineId: "SearchEngineId"); + + // Act - Use generic interface with Contains filtering + KernelSearchResults result = await textSearch.SearchAsync("What is the Semantic Kernel?", + new TextSearchOptions + { + Top = 4, + Skip = 0, + Filter = page => page.Title != null && page.Title.Contains("Semantic") + }); + + // Assert + Assert.NotNull(result); + Assert.NotNull(result.Results); + var resultList = await result.Results.ToListAsync(); + Assert.NotNull(resultList); + Assert.Equal(4, resultList.Count); + } + + [Fact] + public async Task GenericSearchWithEqualityFilterReturnsSuccessfullyAsync() + { + // Arrange + this._messageHandlerStub.AddJsonResponse(File.ReadAllText(WhatIsTheSKResponseJson)); + + using var textSearch = new GoogleTextSearch( + initializer: new() { ApiKey = "ApiKey", HttpClientFactory = this._clientFactory }, + searchEngineId: "SearchEngineId"); + + // Act - Use generic interface with equality filtering + KernelSearchResults result = await textSearch.SearchAsync("What is the Semantic Kernel?", + new TextSearchOptions + { + Top = 4, + Skip = 0, + Filter = page => page.DisplayLink == "microsoft.com" + }); + + // Assert + Assert.NotNull(result); + Assert.NotNull(result.Results); + var resultList = await result.Results.ToListAsync(); + Assert.NotNull(resultList); + Assert.Equal(4, resultList.Count); + } + + [Fact] + public async Task GenericSearchWithNotEqualFilterReturnsSuccessfullyAsync() + { + // Arrange + this._messageHandlerStub.AddJsonResponse(File.ReadAllText(WhatIsTheSKResponseJson)); + + using var textSearch = new GoogleTextSearch( + initializer: new() { ApiKey = "ApiKey", HttpClientFactory = this._clientFactory }, + searchEngineId: "SearchEngineId"); + + // Act - Use generic interface with NOT EQUAL filtering (excludes terms) + KernelSearchResults result = await textSearch.SearchAsync("What is the Semantic Kernel?", + new TextSearchOptions + { + Top = 4, + Skip = 0, + Filter = page => page.Title != "Deprecated" + }); + + // Assert + Assert.NotNull(result); + Assert.NotNull(result.Results); + var resultList = await result.Results.ToListAsync(); + Assert.NotNull(resultList); + Assert.Equal(4, resultList.Count); } + [Fact] + public async Task GenericSearchWithNotContainsFilterReturnsSuccessfullyAsync() + { + // Arrange + this._messageHandlerStub.AddJsonResponse(File.ReadAllText(WhatIsTheSKResponseJson)); + + using var textSearch = new GoogleTextSearch( + initializer: new() { ApiKey = "ApiKey", HttpClientFactory = this._clientFactory }, + searchEngineId: "SearchEngineId"); + + // Act - Use generic interface with NOT Contains filtering (excludes terms) + KernelSearchResults result = await textSearch.SearchAsync("What is the Semantic Kernel?", + new TextSearchOptions + { + Top = 4, + Skip = 0, + Filter = page => page.Title != null && !page.Title.Contains("deprecated") + }); + + // Assert + Assert.NotNull(result); + Assert.NotNull(result.Results); + var resultList = await result.Results.ToListAsync(); + Assert.NotNull(resultList); + Assert.Equal(4, resultList.Count); + } + + [Fact] + public async Task GenericSearchWithFileFormatFilterReturnsSuccessfullyAsync() + { + // Arrange + this._messageHandlerStub.AddJsonResponse(File.ReadAllText(WhatIsTheSKResponseJson)); + + using var textSearch = new GoogleTextSearch( + initializer: new() { ApiKey = "ApiKey", HttpClientFactory = this._clientFactory }, + searchEngineId: "SearchEngineId"); + + // Act - Use generic interface with FileFormat filtering + KernelSearchResults result = await textSearch.SearchAsync("What is the Semantic Kernel?", + new TextSearchOptions + { + Top = 4, + Skip = 0, + Filter = page => page.FileFormat == "pdf" + }); + + // Assert + Assert.NotNull(result); + Assert.NotNull(result.Results); + var resultList = await result.Results.ToListAsync(); + Assert.NotNull(resultList); + Assert.Equal(4, resultList.Count); + } + + [Fact] + public async Task GenericSearchWithCompoundAndFilterReturnsSuccessfullyAsync() + { + // Arrange + this._messageHandlerStub.AddJsonResponse(File.ReadAllText(WhatIsTheSKResponseJson)); + + using var textSearch = new GoogleTextSearch( + initializer: new() { ApiKey = "ApiKey", HttpClientFactory = this._clientFactory }, + searchEngineId: "SearchEngineId"); + + // Act - Use generic interface with compound AND filtering + KernelSearchResults result = await textSearch.SearchAsync("What is the Semantic Kernel?", + new TextSearchOptions + { + Top = 4, + Skip = 0, + Filter = page => page.Title != null && page.Title.Contains("Semantic") && page.DisplayLink != null && page.DisplayLink.Contains("microsoft") + }); + + // Assert + Assert.NotNull(result); + Assert.NotNull(result.Results); + var resultList = await result.Results.ToListAsync(); + Assert.NotNull(resultList); + Assert.Equal(4, resultList.Count); + } + + [Fact] + public async Task GenericSearchWithComplexCompoundFilterReturnsSuccessfullyAsync() + { + // Arrange + this._messageHandlerStub.AddJsonResponse(File.ReadAllText(WhatIsTheSKResponseJson)); + + using var textSearch = new GoogleTextSearch( + initializer: new() { ApiKey = "ApiKey", HttpClientFactory = this._clientFactory }, + searchEngineId: "SearchEngineId"); + + // Act - Use generic interface with complex compound filtering (equality + contains + exclusion) + KernelSearchResults result = await textSearch.SearchAsync("What is the Semantic Kernel?", + new TextSearchOptions + { + Top = 4, + Skip = 0, + Filter = page => page.FileFormat == "pdf" && page.Title != null && page.Title.Contains("AI") && page.Snippet != null && !page.Snippet.Contains("deprecated") + }); + + // Assert + Assert.NotNull(result); + Assert.NotNull(result.Results); + var resultList = await result.Results.ToListAsync(); + Assert.NotNull(resultList); + Assert.Equal(4, resultList.Count); + } + + #region LINQ Filter Verification Tests + // These tests verify that LINQ expressions produce correct Google API URL parameters + // Addressing reviewer feedback: "Some tests to verify the filter url that is created from the different linq expressions would be good" + + [Fact] + public async Task LinqEqualityFilterProducesCorrectApiUrlAsync() + { + // Arrange + this._messageHandlerStub.AddJsonResponse(File.ReadAllText(WhatIsTheSKResponseJson)); + + using var textSearch = new GoogleTextSearch( + initializer: new() { ApiKey = "ApiKey", HttpClientFactory = this._clientFactory }, + searchEngineId: "SearchEngineId"); + + // Act - Use LINQ equality filter for DisplayLink + await textSearch.SearchAsync("test", + new TextSearchOptions + { + Top = 4, + Skip = 0, + Filter = page => page.DisplayLink == "microsoft.com" + }); + + // Assert - Verify URL contains correct siteSearch parameter + var requestUris = this._messageHandlerStub.RequestUris; + Assert.Single(requestUris); + var absoluteUri = requestUris[0]!.AbsoluteUri; + Assert.Contains("siteSearch=microsoft.com", absoluteUri); + Assert.Contains("siteSearchFilter=i", absoluteUri); + } + + [Fact] + public async Task LinqFileFormatEqualityFilterProducesCorrectApiUrlAsync() + { + // Arrange + this._messageHandlerStub.AddJsonResponse(File.ReadAllText(WhatIsTheSKResponseJson)); + + using var textSearch = new GoogleTextSearch( + initializer: new() { ApiKey = "ApiKey", HttpClientFactory = this._clientFactory }, + searchEngineId: "SearchEngineId"); + + // Act - Use LINQ equality filter for FileFormat + await textSearch.SearchAsync("test", + new TextSearchOptions + { + Top = 4, + Skip = 0, + Filter = page => page.FileFormat == "pdf" + }); + + // Assert - Verify URL contains correct fileType parameter + var requestUris = this._messageHandlerStub.RequestUris; + Assert.Single(requestUris); + var absoluteUri = requestUris[0]!.AbsoluteUri; + Assert.Contains("fileType=pdf", absoluteUri); + } + + [Fact] + public async Task LinqContainsFilterProducesCorrectApiUrlAsync() + { + // Arrange + this._messageHandlerStub.AddJsonResponse(File.ReadAllText(WhatIsTheSKResponseJson)); + + using var textSearch = new GoogleTextSearch( + initializer: new() { ApiKey = "ApiKey", HttpClientFactory = this._clientFactory }, + searchEngineId: "SearchEngineId"); + + // Act - Use LINQ Contains filter for Title + await textSearch.SearchAsync("test", + new TextSearchOptions + { + Top = 4, + Skip = 0, + Filter = page => page.Title != null && page.Title.Contains("Semantic") + }); + + // Assert - Verify URL contains correct orTerms parameter (Contains uses orTerms for flexibility) + var requestUris = this._messageHandlerStub.RequestUris; + Assert.Single(requestUris); + var absoluteUri = requestUris[0]!.AbsoluteUri; + Assert.Contains("orTerms=Semantic", absoluteUri); + } + + [Fact] + public async Task LinqNotEqualFilterProducesCorrectApiUrlAsync() + { + // Arrange + this._messageHandlerStub.AddJsonResponse(File.ReadAllText(WhatIsTheSKResponseJson)); + + using var textSearch = new GoogleTextSearch( + initializer: new() { ApiKey = "ApiKey", HttpClientFactory = this._clientFactory }, + searchEngineId: "SearchEngineId"); + + // Act - Use LINQ NOT Equal filter for Title + await textSearch.SearchAsync("test", + new TextSearchOptions + { + Top = 4, + Skip = 0, + Filter = page => page.Title != "deprecated" + }); + + // Assert - Verify URL contains correct excludeTerms parameter + var requestUris = this._messageHandlerStub.RequestUris; + Assert.Single(requestUris); + var absoluteUri = requestUris[0]!.AbsoluteUri; + Assert.Contains("excludeTerms=deprecated", absoluteUri); + } + + [Fact] + public async Task LinqNotContainsFilterProducesCorrectApiUrlAsync() + { + // Arrange + this._messageHandlerStub.AddJsonResponse(File.ReadAllText(WhatIsTheSKResponseJson)); + + using var textSearch = new GoogleTextSearch( + initializer: new() { ApiKey = "ApiKey", HttpClientFactory = this._clientFactory }, + searchEngineId: "SearchEngineId"); + + // Act - Use LINQ NOT Contains filter for Snippet + await textSearch.SearchAsync("test", + new TextSearchOptions + { + Top = 4, + Skip = 0, + Filter = page => page.Snippet != null && !page.Snippet.Contains("outdated") + }); + + // Assert - Verify URL contains correct excludeTerms parameter + var requestUris = this._messageHandlerStub.RequestUris; + Assert.Single(requestUris); + var absoluteUri = requestUris[0]!.AbsoluteUri; + Assert.Contains("excludeTerms=outdated", absoluteUri); + } + + [Fact] + public async Task LinqCompoundAndFilterProducesCorrectApiUrlAsync() + { + // Arrange + this._messageHandlerStub.AddJsonResponse(File.ReadAllText(WhatIsTheSKResponseJson)); + + using var textSearch = new GoogleTextSearch( + initializer: new() { ApiKey = "ApiKey", HttpClientFactory = this._clientFactory }, + searchEngineId: "SearchEngineId"); + + // Act - Use LINQ compound AND filter + await textSearch.SearchAsync("test", + new TextSearchOptions + { + Top = 4, + Skip = 0, + Filter = page => page.DisplayLink == "microsoft.com" && page.FileFormat == "pdf" + }); + + // Assert - Verify URL contains both parameters + var requestUris = this._messageHandlerStub.RequestUris; + Assert.Single(requestUris); + var absoluteUri = requestUris[0]!.AbsoluteUri; + Assert.Contains("siteSearch=microsoft.com", absoluteUri); + Assert.Contains("siteSearchFilter=i", absoluteUri); + Assert.Contains("fileType=pdf", absoluteUri); + } + + [Fact] + public async Task LinqComplexCompoundFilterProducesCorrectApiUrlAsync() + { + // Arrange + this._messageHandlerStub.AddJsonResponse(File.ReadAllText(WhatIsTheSKResponseJson)); + + using var textSearch = new GoogleTextSearch( + initializer: new() { ApiKey = "ApiKey", HttpClientFactory = this._clientFactory }, + searchEngineId: "SearchEngineId"); + + // Act - Use LINQ complex compound filter (equality + contains + exclusion) + await textSearch.SearchAsync("test", + new TextSearchOptions + { + Top = 4, + Skip = 0, + Filter = page => page.FileFormat == "pdf" && + page.Title != null && page.Title.Contains("AI") && + page.Snippet != null && !page.Snippet.Contains("deprecated") + }); + + // Assert - Verify URL contains all expected parameters + var requestUris = this._messageHandlerStub.RequestUris; + Assert.Single(requestUris); + var absoluteUri = requestUris[0]!.AbsoluteUri; + Assert.Contains("fileType=pdf", absoluteUri); + Assert.Contains("orTerms=AI", absoluteUri); // Contains uses orTerms for flexibility + Assert.Contains("excludeTerms=deprecated", absoluteUri); + } + + [Fact] + public async Task CollectionContainsFilterThrowsNotSupportedExceptionAsync() + { + // Arrange + using var textSearch = new GoogleTextSearch( + initializer: new() { ApiKey = "ApiKey", HttpClientFactory = this._clientFactory }, + searchEngineId: "SearchEngineId"); + + // Act & Assert - Collection Contains (both Enumerable.Contains and MemoryExtensions.Contains) + // This same code resolves differently based on C# language version: + // - C# 13 and earlier: Enumerable.Contains (LINQ extension method) + // - C# 14 and later: MemoryExtensions.Contains (span-based optimization) + // Our implementation handles both identically - both throw NotSupportedException + string[] sites = ["microsoft.com", "github.com"]; + var exception = await Assert.ThrowsAsync(async () => + await textSearch.SearchAsync("test", + new TextSearchOptions + { + Top = 4, + Skip = 0, + Filter = page => sites.Contains(page.DisplayLink!) + })); + + // Verify exception message is clear and actionable + Assert.Contains("Collection Contains filters", exception.Message); + Assert.Contains("not supported by Google Custom Search API", exception.Message); + Assert.Contains("OR logic", exception.Message); + } + + #endregion + /// public void Dispose() { diff --git a/dotnet/src/Plugins/Plugins.UnitTests/Web/Tavily/TavilyTextSearchTests.cs b/dotnet/src/Plugins/Plugins.UnitTests/Web/Tavily/TavilyTextSearchTests.cs index 553290a4287d..c51dbb769e34 100644 --- a/dotnet/src/Plugins/Plugins.UnitTests/Web/Tavily/TavilyTextSearchTests.cs +++ b/dotnet/src/Plugins/Plugins.UnitTests/Web/Tavily/TavilyTextSearchTests.cs @@ -1,5 +1,8 @@ // Copyright (c) Microsoft. All rights reserved. +#pragma warning disable CS0618 // ITextSearch is obsolete +#pragma warning disable CS8602 // Dereference of a possibly null reference - for LINQ expression properties + using System; using System.IO; using System.Linq; @@ -344,6 +347,156 @@ public void Dispose() GC.SuppressFinalize(this); } + #region Generic ITextSearch Interface Tests + + [Fact] + public async Task LinqSearchAsyncReturnsResultsSuccessfullyAsync() + { + // Arrange + this._messageHandlerStub.AddJsonResponse(File.ReadAllText(SiteFilterDevBlogsResponseJson)); + ITextSearch textSearch = new TavilyTextSearch(apiKey: "ApiKey", options: new() { HttpClient = this._httpClient }); + + // Act + var searchOptions = new TextSearchOptions + { + Top = 4, + Skip = 0 + }; + KernelSearchResults result = await textSearch.SearchAsync("What is the Semantic Kernel?", searchOptions); + + // Assert - Verify basic generic interface functionality + Assert.NotNull(result); + Assert.NotNull(result.Results); + var resultList = await result.Results.ToListAsync(); + Assert.NotEmpty(resultList); + + // Verify the request was made correctly + var requestContents = this._messageHandlerStub.RequestContents; + Assert.Single(requestContents); + Assert.NotNull(requestContents[0]); + var requestBodyJson = Encoding.UTF8.GetString(requestContents[0]!); + Assert.Contains("\"query\"", requestBodyJson); + Assert.Contains("\"max_results\":4", requestBodyJson); + } + + [Fact] + public async Task LinqGetSearchResultsAsyncReturnsResultsSuccessfullyAsync() + { + // Arrange + this._messageHandlerStub.AddJsonResponse(File.ReadAllText(SiteFilterDevBlogsResponseJson)); + ITextSearch textSearch = new TavilyTextSearch(apiKey: "ApiKey", options: new() { HttpClient = this._httpClient }); + + // Act + var searchOptions = new TextSearchOptions + { + Top = 3, + Skip = 0 + }; + KernelSearchResults result = await textSearch.GetSearchResultsAsync("What is the Semantic Kernel?", searchOptions); + + // Assert - Verify generic interface returns results + Assert.NotNull(result); + Assert.NotNull(result.Results); + var resultList = await result.Results.ToListAsync(); + Assert.NotEmpty(resultList); + // Results are now strongly typed as TavilyWebPage + + // Verify the request was made correctly + var requestContents = this._messageHandlerStub.RequestContents; + Assert.Single(requestContents); + Assert.NotNull(requestContents[0]); + var requestBodyJson = Encoding.UTF8.GetString(requestContents[0]!); + Assert.Contains("\"max_results\":3", requestBodyJson); + } + + [Fact] + public async Task LinqGetTextSearchResultsAsyncReturnsResultsSuccessfullyAsync() + { + // Arrange + this._messageHandlerStub.AddJsonResponse(File.ReadAllText(SiteFilterDevBlogsResponseJson)); + ITextSearch textSearch = new TavilyTextSearch(apiKey: "ApiKey", options: new() { HttpClient = this._httpClient }); + + // Act + var searchOptions = new TextSearchOptions + { + Top = 5, + Skip = 0 + }; + KernelSearchResults result = await textSearch.GetTextSearchResultsAsync("What is the Semantic Kernel?", searchOptions); + + // Assert - Verify generic interface returns TextSearchResult objects + Assert.NotNull(result); + Assert.NotNull(result.Results); + var resultList = await result.Results.ToListAsync(); + Assert.NotEmpty(resultList); + Assert.All(resultList, item => Assert.IsType(item)); + + // Verify the request was made correctly + var requestContents = this._messageHandlerStub.RequestContents; + Assert.Single(requestContents); + Assert.NotNull(requestContents[0]); + var requestBodyJson = Encoding.UTF8.GetString(requestContents[0]!); + Assert.Contains("\"max_results\":5", requestBodyJson); + } + + [Fact] + public async Task CollectionContainsFilterThrowsNotSupportedExceptionAsync() + { + // Arrange - Tests both Enumerable.Contains (C# 13-) and MemoryExtensions.Contains (C# 14+) + // The same code array.Contains() resolves differently based on C# language version: + // - C# 13 and earlier: Enumerable.Contains (LINQ extension method) + // - C# 14 and later: MemoryExtensions.Contains (span-based optimization due to "first-class spans") + // Our implementation handles both identically since Tavily API has limited query operators + this._messageHandlerStub.AddJsonResponse(File.ReadAllText(SiteFilterDevBlogsResponseJson)); + ITextSearch textSearch = new TavilyTextSearch(apiKey: "ApiKey", options: new() { HttpClient = this._httpClient }); + string[] domains = ["microsoft.com", "github.com"]; + + // Act & Assert - Verify that collection Contains pattern throws clear exception + var searchOptions = new TextSearchOptions + { + Top = 5, + Skip = 0, + Filter = page => domains.Contains(page.Url!.ToString()) // Enumerable.Contains (C# 13-) or MemoryExtensions.Contains (C# 14+) + }; + + var exception = await Assert.ThrowsAsync(async () => + { + await textSearch.SearchAsync("test", searchOptions); + }); + + // Assert - Verify error message explains the limitation clearly + Assert.Contains("Collection Contains filters", exception.Message); + Assert.Contains("not supported", exception.Message); + } + + [Fact] + public async Task StringContainsStillWorksWithLINQFiltersAsync() + { + // Arrange - Verify that String.Contains (instance method) still works + // String.Contains is NOT affected by C# 14 "first-class spans" - only arrays are + this._messageHandlerStub.AddJsonResponse(File.ReadAllText(SiteFilterDevBlogsResponseJson)); + ITextSearch textSearch = new TavilyTextSearch(apiKey: "ApiKey", options: new() { HttpClient = this._httpClient }); + + // Act - String.Contains should continue to work + var searchOptions = new TextSearchOptions + { + Top = 5, + Skip = 0, + Filter = page => page.Title.Contains("Kernel") // String.Contains - instance method + }; + KernelSearchResults result = await textSearch.SearchAsync("Semantic Kernel tutorial", searchOptions); + + // Assert - Verify String.Contains works correctly + var requestContents = this._messageHandlerStub.RequestContents; + Assert.Single(requestContents); + Assert.NotNull(requestContents[0]); + var requestBodyJson = Encoding.UTF8.GetString(requestContents[0]!); + Assert.Contains("Kernel", requestBodyJson); + Assert.Contains("\"max_results\":5", requestBodyJson); + } + + #endregion + #region private private const string WhatIsTheSKResponseJson = "./TestData/tavily_what_is_the_semantic_kernel.json"; private const string SiteFilterDevBlogsResponseJson = "./TestData/tavily_site_filter_devblogs_microsoft.com.json"; diff --git a/dotnet/src/Plugins/Plugins.Web/Bing/BingTextSearch.cs b/dotnet/src/Plugins/Plugins.Web/Bing/BingTextSearch.cs index 556e04f148d3..2dc07d8b4422 100644 --- a/dotnet/src/Plugins/Plugins.Web/Bing/BingTextSearch.cs +++ b/dotnet/src/Plugins/Plugins.Web/Bing/BingTextSearch.cs @@ -1,8 +1,11 @@ // Copyright (c) Microsoft. All rights reserved. +#pragma warning disable CS0618 // Non-generic ITextSearch is obsolete - provides backward compatibility during Phase 2 LINQ migration + using System; using System.Collections.Generic; using System.Linq; +using System.Linq.Expressions; using System.Net.Http; using System.Runtime.CompilerServices; using System.Text; @@ -20,7 +23,9 @@ namespace Microsoft.SemanticKernel.Plugins.Web.Bing; /// /// A Bing Text Search implementation that can be used to perform searches using the Bing Web Search API. /// -public sealed class BingTextSearch : ITextSearch +#pragma warning disable CS0618 // ITextSearch is obsolete - this class provides backward compatibility +public sealed class BingTextSearch : ITextSearch, ITextSearch +#pragma warning restore CS0618 { /// /// Create an instance of the with API key authentication. @@ -74,6 +79,35 @@ public async Task> GetSearchResultsAsync(string quer return new KernelSearchResults(this.GetResultsAsWebPageAsync(searchResponse, cancellationToken), totalCount, GetResultsMetadata(searchResponse)); } + /// + Task> ITextSearch.SearchAsync(string query, TextSearchOptions? searchOptions, CancellationToken cancellationToken) + { + var legacyOptions = searchOptions != null ? ConvertToLegacyOptions(searchOptions) : new TextSearchOptions(); + return this.SearchAsync(query, legacyOptions, cancellationToken); + } + + /// + Task> ITextSearch.GetTextSearchResultsAsync(string query, TextSearchOptions? searchOptions, CancellationToken cancellationToken) + { + var legacyOptions = searchOptions != null ? ConvertToLegacyOptions(searchOptions) : new TextSearchOptions(); + return this.GetTextSearchResultsAsync(query, legacyOptions, cancellationToken); + } + + /// + async Task> ITextSearch.GetSearchResultsAsync(string query, TextSearchOptions? searchOptions, CancellationToken cancellationToken) + { + var legacyOptions = searchOptions != null ? ConvertToLegacyOptions(searchOptions) : new TextSearchOptions(); + + BingSearchResponse? searchResponse = await this.ExecuteSearchAsync(query, legacyOptions, cancellationToken).ConfigureAwait(false); + + long? totalCount = legacyOptions.IncludeTotalCount ? searchResponse?.WebPages?.TotalEstimatedMatches : null; + + return new KernelSearchResults( + this.GetResultsAsBingWebPageAsync(searchResponse, cancellationToken), + totalCount, + GetResultsMetadata(searchResponse)); + } + #region private private readonly ILogger _logger; @@ -92,6 +126,293 @@ public async Task> GetSearchResultsAsync(string quer private const string DefaultUri = "https://api.bing.microsoft.com/v7.0/search"; + /// + /// Converts generic TextSearchOptions with LINQ filtering to legacy TextSearchOptions. + /// Attempts to translate simple LINQ expressions to Bing API filters where possible. + /// + /// The generic search options with LINQ filtering. + /// Legacy TextSearchOptions with equivalent filtering, or null if no conversion possible. + private static TextSearchOptions ConvertToLegacyOptions(TextSearchOptions genericOptions) + { + return new TextSearchOptions + { + Top = genericOptions.Top, + Skip = genericOptions.Skip, + Filter = genericOptions.Filter != null ? ConvertLinqExpressionToBingFilter(genericOptions.Filter) : null + }; + } + + /// + /// Converts a LINQ expression to a TextSearchFilter compatible with Bing API. + /// Supports equality, inequality, Contains() method calls, and logical AND operator. + /// + /// The LINQ expression to convert. + /// A TextSearchFilter with equivalent filtering. + /// Thrown when the expression cannot be converted to Bing filters. + private static TextSearchFilter ConvertLinqExpressionToBingFilter(Expression> linqExpression) + { + var filter = new TextSearchFilter(); + ProcessExpression(linqExpression.Body, filter); + return filter; + } + + /// + /// Recursively processes LINQ expression nodes and builds Bing API filters. + /// + private static void ProcessExpression(Expression expression, TextSearchFilter filter) + { + switch (expression) + { + case BinaryExpression binaryExpr when binaryExpr.NodeType == ExpressionType.AndAlso: + // Handle AND: page => page.Language == "en" && page.Name.Contains("AI") + ProcessExpression(binaryExpr.Left, filter); + ProcessExpression(binaryExpr.Right, filter); + break; + + case BinaryExpression binaryExpr when binaryExpr.NodeType == ExpressionType.OrElse: + // Handle OR: Currently not directly supported by TextSearchFilter + // Bing API supports OR via multiple queries, but TextSearchFilter doesn't expose this + throw new NotSupportedException( + "Logical OR (||) is not supported by Bing Text Search filters. " + + "Consider splitting into multiple search queries."); + + case UnaryExpression unaryExpr when unaryExpr.NodeType == ExpressionType.Not: + // Handle NOT: page => !page.Language.Equals("en") + throw new NotSupportedException( + "Logical NOT (!) is not directly supported by Bing Text Search advanced operators. " + + "Consider restructuring your filter to use positive conditions."); + + case BinaryExpression binaryExpr when binaryExpr.NodeType == ExpressionType.Equal: + // Handle equality: page => page.Language == "en" + ProcessEqualityExpression(binaryExpr, filter, isNegated: false); + break; + + case BinaryExpression binaryExpr when binaryExpr.NodeType == ExpressionType.NotEqual: + // Handle inequality: page => page.Language != "en" + // Implemented via Bing's negation syntax (e.g., -language:en) + ProcessEqualityExpression(binaryExpr, filter, isNegated: true); + break; + + case MethodCallExpression methodExpr when methodExpr.Method.Name == "Contains": + // Distinguish between instance method (String.Contains) and static method (Enumerable/MemoryExtensions.Contains) + if (methodExpr.Object is MemberExpression) + { + // Instance method: page.Name.Contains("value") - SUPPORTED + ProcessContainsExpression(methodExpr, filter); + } + else if (methodExpr.Object == null) + { + // Static method: could be Enumerable.Contains (C# 13-) or MemoryExtensions.Contains (C# 14+) + // Bing API doesn't support OR logic, so collection Contains patterns are not supported + if (methodExpr.Method.DeclaringType == typeof(Enumerable) || + (methodExpr.Method.DeclaringType == typeof(MemoryExtensions) && IsMemoryExtensionsContains(methodExpr))) + { + throw new NotSupportedException( + "Collection Contains filters (e.g., array.Contains(page.Property)) are not supported by Bing Search API. " + + "Bing's advanced search operators do not support OR logic across multiple values. " + + "Supported pattern: Property.Contains(\"value\") for string properties like Name, Snippet, or Url. " + + "For multiple value matching, consider alternative approaches or use a different search provider."); + } + + throw new NotSupportedException( + $"Contains() method from {methodExpr.Method.DeclaringType?.Name} is not supported."); + } + else + { + throw new NotSupportedException( + "Contains() must be called on a property (e.g., page.Name.Contains(\"value\"))."); + } + break; + + default: + throw new NotSupportedException( + $"Expression type '{expression.NodeType}' is not supported for Bing API filters. " + + "Supported patterns: equality (==), inequality (!=), Contains(), and logical AND (&&). " + + "Available Bing operators: " + string.Join(", ", s_advancedSearchKeywords)); + } + } + + /// + /// Processes equality and inequality expressions (property == value or property != value). + /// + /// The binary expression to process. + /// The filter to update. + /// True if this is an inequality (!=) expression. + private static void ProcessEqualityExpression(BinaryExpression binaryExpr, TextSearchFilter filter, bool isNegated) + { + // Handle nullable properties with conversions (e.g., bool? == bool becomes Convert(property) == value) + MemberExpression? memberExpr = binaryExpr.Left as MemberExpression; + if (memberExpr == null && binaryExpr.Left is UnaryExpression unaryExpr && unaryExpr.NodeType == ExpressionType.Convert) + { + memberExpr = unaryExpr.Operand as MemberExpression; + } + + // Handle conversions on the right side too + ConstantExpression? constExpr = binaryExpr.Right as ConstantExpression; + if (constExpr == null && binaryExpr.Right is UnaryExpression rightUnaryExpr && rightUnaryExpr.NodeType == ExpressionType.Convert) + { + constExpr = rightUnaryExpr.Operand as ConstantExpression; + } + + if (memberExpr != null && constExpr != null) + { + string propertyName = memberExpr.Member.Name; + object? value = constExpr.Value; + + string? bingFilterName = MapPropertyToBingFilter(propertyName); + if (bingFilterName != null && value != null) + { + // Convert boolean values to lowercase strings for Bing API compatibility + // CA1308: Using ToLowerInvariant() is intentional here as Bing API expects boolean values in lowercase format (true/false) +#pragma warning disable CA1308 // Normalize strings to uppercase + string stringValue = value is bool boolValue ? boolValue.ToString().ToLowerInvariant() : value.ToString() ?? string.Empty; +#pragma warning restore CA1308 // Normalize strings to uppercase + + if (isNegated) + { + // For inequality, wrap the value with a negation marker + // This will be processed in BuildQuery to prepend '-' to the advanced search operator + // Example: language:en becomes -language:en (excludes pages in English) + filter.Equality(bingFilterName, $"-{stringValue}"); + } + else + { + filter.Equality(bingFilterName, stringValue); + } + } + else if (value == null) + { + throw new NotSupportedException( + $"Null values are not supported in Bing API filters for property '{propertyName}'."); + } + else + { + throw new NotSupportedException( + $"Property '{propertyName}' cannot be mapped to Bing API filters. " + + "Supported properties: Language, Url, DisplayUrl, Name, Snippet, IsFamilyFriendly."); + } + } + else + { + throw new NotSupportedException( + "Equality expressions must be in the form 'property == value' or 'property != value'. " + + "Complex expressions on the left or right side are not supported."); + } + } + + /// + /// Processes Contains() method calls on string properties. + /// Maps to Bing's advanced search operators like intitle:, inbody:, url:. + /// + private static void ProcessContainsExpression(MethodCallExpression methodExpr, TextSearchFilter filter) + { + // Contains can be called on a property: page.Name.Contains("value") + // or on a collection: page.Tags.Contains("value") + + if (methodExpr.Object is MemberExpression memberExpr) + { + string propertyName = memberExpr.Member.Name; + + // Extract the search value from the Contains() argument + if (methodExpr.Arguments.Count == 1 && methodExpr.Arguments[0] is ConstantExpression constExpr) + { + object? value = constExpr.Value; + if (value == null) + { + return; // Skip null values + } + + // Map property to Bing filter with Contains semantic + string? bingFilterOperator = MapPropertyToContainsFilter(propertyName); + if (bingFilterOperator != null) + { + // Use Bing's advanced search syntax: intitle:"value", inbody:"value", etc. + filter.Equality(bingFilterOperator, value); + } + else + { + throw new NotSupportedException( + $"Contains() on property '{propertyName}' is not supported by Bing API filters. " + + "Supported properties for Contains: Name (maps to intitle:), Snippet (maps to inbody:), Url (maps to url:)."); + } + } + else + { + throw new NotSupportedException( + "Contains() must have a single constant value argument. " + + "Complex expressions as arguments are not supported."); + } + } + else + { + throw new NotSupportedException( + "Contains() must be called on a property (e.g., page.Name.Contains(\"value\")). " + + "Collection Contains patterns are not yet supported."); + } + } + + /// + /// Determines if a MethodCallExpression is a MemoryExtensions.Contains call (C# 14 "first-class spans" feature). + /// + /// The method call expression to check. + /// True if this is a MemoryExtensions.Contains call with supported parameters; otherwise false. + private static bool IsMemoryExtensionsContains(MethodCallExpression methodExpr) + { + // MemoryExtensions.Contains has 2-3 parameters: + // - Contains(ReadOnlySpan span, T value) + // - Contains(ReadOnlySpan span, T value, IEqualityComparer? comparer) + // We only support when comparer is null or omitted + return methodExpr.Method.Name == nameof(MemoryExtensions.Contains) && + methodExpr.Arguments.Count >= 2 && + methodExpr.Arguments.Count <= 3 && + (methodExpr.Arguments.Count == 2 || + (methodExpr.Arguments.Count == 3 && methodExpr.Arguments[2] is ConstantExpression { Value: null })); + } + + /// + /// Maps BingWebPage property names to Bing API filter field names for equality operations. + /// + /// The BingWebPage property name. + /// The corresponding Bing API filter name, or null if not mappable. + private static string? MapPropertyToBingFilter(string propertyName) + { + return propertyName.ToUpperInvariant() switch + { + // Map BingWebPage properties to Bing API equivalents + "LANGUAGE" => "language", // Maps to advanced search + "URL" => "url", // Maps to advanced search + "DISPLAYURL" => "site", // Maps to site: search + "NAME" => "intitle", // Maps to title search + "SNIPPET" => "inbody", // Maps to body content search + "ISFAMILYFRIENDLY" => "safeSearch", // Maps to safe search parameter + + // Direct API parameters (if we ever extend BingWebPage with metadata) + "MKT" => "mkt", // Market/locale + "FRESHNESS" => "freshness", // Date freshness + + _ => null // Property not mappable to Bing filters + }; + } + + /// + /// Maps BingWebPage property names to Bing API advanced search operators for Contains operations. + /// + /// The BingWebPage property name. + /// The corresponding Bing advanced search operator, or null if not mappable. + private static string? MapPropertyToContainsFilter(string propertyName) + { + return propertyName.ToUpperInvariant() switch + { + // Map properties to Bing's contains-style operators + "NAME" => "intitle", // intitle:"search term" - title contains + "SNIPPET" => "inbody", // inbody:"search term" - body contains + "URL" => "url", // url:"search term" - URL contains + "DISPLAYURL" => "site", // site:domain.com - site contains + + _ => null // Property not mappable to Contains-style filters + }; + } + /// /// Execute a Bing search query and return the results. /// @@ -139,7 +460,7 @@ private async Task SendGetRequestAsync(string query, TextSe } /// - /// Return the search results as instances of . + /// Return the search results as instances of . /// /// Response containing the web pages matching the query. /// Cancellation token @@ -157,6 +478,25 @@ private async IAsyncEnumerable GetResultsAsWebPageAsync(BingSearchRespon } } + /// + /// Return the search results as strongly-typed instances. + /// + /// Response containing the web pages matching the query. + /// Cancellation token + private async IAsyncEnumerable GetResultsAsBingWebPageAsync(BingSearchResponse? searchResponse, [EnumeratorCancellation] CancellationToken cancellationToken) + { + if (searchResponse is null || searchResponse.WebPages is null || searchResponse.WebPages.Value is null) + { + yield break; + } + + foreach (var webPage in searchResponse.WebPages.Value) + { + yield return webPage; + await Task.Yield(); + } + } + /// /// Return the search results as instances of . /// @@ -260,14 +600,21 @@ private static string BuildQuery(string query, TextSearchOptions searchOptions) { if (filterClause is EqualToFilterClause equalityFilterClause) { + // Check if value starts with '-' indicating negation (for inequality != operator) + string? valueStr = equalityFilterClause.Value?.ToString(); + bool isNegated = valueStr?.StartsWith("-", StringComparison.Ordinal) == true; + string actualValue = isNegated && valueStr != null ? valueStr.Substring(1) : valueStr ?? string.Empty; + if (s_advancedSearchKeywords.Contains(equalityFilterClause.FieldName, StringComparer.OrdinalIgnoreCase) && equalityFilterClause.Value is not null) { - fullQuery.Append($"+{equalityFilterClause.FieldName}%3A").Append(Uri.EscapeDataString(equalityFilterClause.Value.ToString()!)); + // For advanced search keywords, prepend '-' if negated to exclude results + string prefix = isNegated ? "-" : ""; + fullQuery.Append($"+{prefix}{equalityFilterClause.FieldName}%3A").Append(Uri.EscapeDataString(actualValue)); } else if (s_queryParameters.Contains(equalityFilterClause.FieldName, StringComparer.OrdinalIgnoreCase) && equalityFilterClause.Value is not null) { string? queryParam = s_queryParameters.FirstOrDefault(s => s.Equals(equalityFilterClause.FieldName, StringComparison.OrdinalIgnoreCase)); - queryParams.Append('&').Append(queryParam!).Append('=').Append(Uri.EscapeDataString(equalityFilterClause.Value.ToString()!)); + queryParams.Append('&').Append(queryParam!).Append('=').Append(Uri.EscapeDataString(actualValue)); } else { diff --git a/dotnet/src/Plugins/Plugins.Web/Brave/BraveTextSearch.cs b/dotnet/src/Plugins/Plugins.Web/Brave/BraveTextSearch.cs index 8fa793ea4efb..e7b6eab6f780 100644 --- a/dotnet/src/Plugins/Plugins.Web/Brave/BraveTextSearch.cs +++ b/dotnet/src/Plugins/Plugins.Web/Brave/BraveTextSearch.cs @@ -3,6 +3,7 @@ using System; using System.Collections.Generic; using System.Linq; +using System.Linq.Expressions; using System.Net.Http; using System.Runtime.CompilerServices; using System.Text; @@ -20,7 +21,9 @@ namespace Microsoft.SemanticKernel.Plugins.Web.Brave; /// /// A Brave Text Search implementation that can be used to perform searches using the Brave Web Search API. /// -public sealed class BraveTextSearch : ITextSearch +#pragma warning disable CS0618 // ITextSearch is obsolete - this class provides backward compatibility +public sealed class BraveTextSearch : ITextSearch, ITextSearch +#pragma warning restore CS0618 { /// /// Create an instance of the with API key authentication. @@ -75,10 +78,438 @@ public async Task> GetSearchResultsAsync(string quer long? totalCount = searchOptions.IncludeTotalCount ? searchResponse?.Web?.Results.Count : null; - return new KernelSearchResults(this.GetResultsAsWebPageAsync(searchResponse, cancellationToken), totalCount, GetResultsMetadata(searchResponse)); + return new KernelSearchResults(this.GetResultsAsObjectAsync(searchResponse, cancellationToken), totalCount, GetResultsMetadata(searchResponse)); } - #region private + #region Generic ITextSearch Implementation + + /// + async Task> ITextSearch.SearchAsync(string query, TextSearchOptions? searchOptions, CancellationToken cancellationToken) + { + var (modifiedQuery, legacyOptions) = this.ConvertToLegacyOptionsWithQuery(query, searchOptions); + BraveSearchResponse? searchResponse = await this.ExecuteSearchAsync(modifiedQuery, legacyOptions, cancellationToken).ConfigureAwait(false); + + long? totalCount = legacyOptions.IncludeTotalCount ? searchResponse?.Web?.Results.Count : null; + + return new KernelSearchResults(this.GetResultsAsStringAsync(searchResponse, cancellationToken), totalCount, GetResultsMetadata(searchResponse)); + } + + /// + async Task> ITextSearch.GetTextSearchResultsAsync(string query, TextSearchOptions? searchOptions, CancellationToken cancellationToken) + { + var (modifiedQuery, legacyOptions) = this.ConvertToLegacyOptionsWithQuery(query, searchOptions); + BraveSearchResponse? searchResponse = await this.ExecuteSearchAsync(modifiedQuery, legacyOptions, cancellationToken).ConfigureAwait(false); + + long? totalCount = legacyOptions.IncludeTotalCount ? searchResponse?.Web?.Results.Count : null; + + return new KernelSearchResults(this.GetResultsAsTextSearchResultAsync(searchResponse, cancellationToken), totalCount, GetResultsMetadata(searchResponse)); + } + + /// + async Task> ITextSearch.GetSearchResultsAsync(string query, TextSearchOptions? searchOptions, CancellationToken cancellationToken) + { + var (modifiedQuery, legacyOptions) = this.ConvertToLegacyOptionsWithQuery(query, searchOptions); + BraveSearchResponse? searchResponse = await this.ExecuteSearchAsync(modifiedQuery, legacyOptions, cancellationToken).ConfigureAwait(false); + + long? totalCount = legacyOptions.IncludeTotalCount ? searchResponse?.Web?.Results.Count : null; + + return new KernelSearchResults(this.GetResultsAsBraveWebPageAsync(searchResponse, cancellationToken), totalCount, GetResultsMetadata(searchResponse)); + } + + #endregion + + #region LINQ-to-Brave Conversion Logic + + /// + /// Converts generic TextSearchOptions with LINQ filtering to legacy TextSearchOptions and extracts additional search terms. + /// + /// The original search query. + /// The generic search options with LINQ filter. + /// A tuple containing the modified query and legacy TextSearchOptions with converted filters. + private (string modifiedQuery, TextSearchOptions legacyOptions) ConvertToLegacyOptionsWithQuery(string query, TextSearchOptions? options) + { + var legacyOptions = this.ConvertToLegacyOptions(options); + + if (options?.Filter != null) + { + // Extract search terms from the LINQ expression + var additionalSearchTerms = ExtractSearchTermsFromLinqExpression(options.Filter); + if (additionalSearchTerms.Count > 0) + { + // Append additional search terms to the original query + var modifiedQuery = $"{query} {string.Join(" ", additionalSearchTerms)}".Trim(); + return (modifiedQuery, legacyOptions); + } + } + + return (query, legacyOptions); + } + + /// + /// Converts generic TextSearchOptions with LINQ filtering to legacy TextSearchOptions. + /// + /// The generic search options with LINQ filter. + /// Legacy TextSearchOptions with converted filters. + private TextSearchOptions ConvertToLegacyOptions(TextSearchOptions? options) + { + if (options == null) + { + return new TextSearchOptions(); + } + + var legacyOptions = new TextSearchOptions + { + Top = options.Top, + Skip = options.Skip, + IncludeTotalCount = options.IncludeTotalCount + }; + + // Convert LINQ expression to TextSearchFilter if present + if (options.Filter != null) + { + try + { + var convertedFilter = ConvertLinqExpressionToBraveFilter(options.Filter); + legacyOptions = new TextSearchOptions + { + Top = options.Top, + Skip = options.Skip, + IncludeTotalCount = options.IncludeTotalCount, + Filter = convertedFilter + }; + } + catch (NotSupportedException) + { + // All unsupported LINQ patterns should fail explicitly to provide clear developer feedback + // This helps developers understand which patterns work with the Brave API + throw; + } + } + + return legacyOptions; + } + + /// + /// Extracts search terms that should be added to the search query from a LINQ expression. + /// + /// The LINQ expression to analyze. + /// A list of search terms to add to the query. + private static List ExtractSearchTermsFromLinqExpression(Expression> linqExpression) + { + var searchTerms = new List(); + var filterClauses = new List(); + + // Analyze the LINQ expression to get all filter clauses + AnalyzeExpression(linqExpression.Body, filterClauses); + + // Extract search terms from SearchQueryFilterClause instances + foreach (var clause in filterClauses) + { + if (clause is SearchQueryFilterClause searchQueryClause) + { + searchTerms.Add(searchQueryClause.SearchTerm); + } + } + + return searchTerms; + } + + /// + /// Converts a LINQ expression to Brave-compatible TextSearchFilter. + /// + /// The LINQ expression to convert. + /// A TextSearchFilter with Brave-compatible filter clauses. + private static TextSearchFilter ConvertLinqExpressionToBraveFilter(Expression> linqExpression) + { + var filter = new TextSearchFilter(); + var filterClauses = new List(); + + // Analyze the LINQ expression and convert to filter clauses + AnalyzeExpression(linqExpression.Body, filterClauses); + + // Validate and add clauses that are supported by Brave + foreach (var clause in filterClauses) + { + if (clause is EqualToFilterClause equalityClause) + { + var mappedFieldName = MapPropertyToBraveFilter(equalityClause.FieldName); + if (mappedFieldName != null) + { + filter.Equality(mappedFieldName, equalityClause.Value); + } + else + { + throw new NotSupportedException( + $"Property '{equalityClause.FieldName}' cannot be mapped to Brave API filters. " + + $"Supported properties: {string.Join(", ", s_queryParameters)}. " + + "Example: page => page.Country == \"US\" && page.SafeSearch == \"moderate\""); + } + } + else if (clause is SearchQueryFilterClause) + { + // SearchQueryFilterClause is handled at the query level, not the filter level + // Skip it here as it's processed by ConvertToLegacyOptionsWithQuery + continue; + } + } + + return filter; + } + + /// + /// Maps BraveWebPage property names to Brave API filter parameter names. + /// + /// The property name from BraveWebPage. + /// The corresponding Brave API parameter name, or null if not mappable. + private static string? MapPropertyToBraveFilter(string propertyName) => + propertyName.ToUpperInvariant() switch + { + "COUNTRY" => BraveParamCountry, + "SEARCHLANG" => BraveParamSearchLang, + "UILANG" => BraveParamUiLang, + "SAFESEARCH" => BraveParamSafeSearch, + "TEXTDECORATIONS" => BraveParamTextDecorations, + "SPELLCHECK" => BraveParamSpellCheck, + "RESULTFILTER" => BraveParamResultFilter, + "UNITS" => BraveParamUnits, + "EXTRASNIPPETS" => BraveParamExtraSnippets, + _ => null // Property not mappable to Brave filters + }; + + // TODO: Consider extracting LINQ expression analysis logic to a shared utility class + // to reduce duplication across text search connectors (Brave, Tavily, etc.). + // See code review for details. + /// + /// Analyzes a LINQ expression and extracts filter clauses. + /// + /// The expression to analyze. + /// The list to add extracted filter clauses to. + private static void AnalyzeExpression(Expression expression, List filterClauses) + { + switch (expression) + { + case BinaryExpression binaryExpr: + if (binaryExpr.NodeType == ExpressionType.AndAlso) + { + // Handle AND expressions by recursively analyzing both sides + AnalyzeExpression(binaryExpr.Left, filterClauses); + AnalyzeExpression(binaryExpr.Right, filterClauses); + } + else if (binaryExpr.NodeType == ExpressionType.OrElse) + { + // Handle OR expressions by recursively analyzing both sides + // Note: OR results in multiple filter values for the same property + AnalyzeExpression(binaryExpr.Left, filterClauses); + AnalyzeExpression(binaryExpr.Right, filterClauses); + } + else if (binaryExpr.NodeType == ExpressionType.Equal) + { + // Handle equality expressions + ExtractEqualityClause(binaryExpr, filterClauses); + } + else if (binaryExpr.NodeType == ExpressionType.NotEqual) + { + // Handle inequality expressions (property != value) + // This is supported as a negation pattern + ExtractInequalityClause(binaryExpr, filterClauses); + } + else + { + throw new NotSupportedException($"Binary expression type '{binaryExpr.NodeType}' is not supported. Supported operators: AndAlso (&&), OrElse (||), Equal (==), NotEqual (!=)."); + } + break; + + case UnaryExpression unaryExpr when unaryExpr.NodeType == ExpressionType.Not: + // Handle NOT expressions (negation) + AnalyzeNotExpression(unaryExpr, filterClauses); + break; + + case MethodCallExpression methodCall: + // Handle method calls like Contains, StartsWith, etc. + ExtractMethodCallClause(methodCall, filterClauses); + break; + + default: + throw new NotSupportedException($"Expression type '{expression.NodeType}' is not supported in Brave search filters."); + } + } + + /// + /// Extracts an equality filter clause from a binary equality expression. + /// + /// The binary equality expression. + /// The list to add the extracted clause to. + private static void ExtractEqualityClause(BinaryExpression binaryExpr, List filterClauses) + { + string? propertyName = null; + object? value = null; + + // Determine which side is the property and which is the value + if (binaryExpr.Left is MemberExpression leftMember) + { + propertyName = leftMember.Member.Name; + value = ExtractValue(binaryExpr.Right); + } + else if (binaryExpr.Right is MemberExpression rightMember) + { + propertyName = rightMember.Member.Name; + value = ExtractValue(binaryExpr.Left); + } + + if (propertyName != null && value != null) + { + filterClauses.Add(new EqualToFilterClause(propertyName, value)); + } + else + { + throw new NotSupportedException("Unable to extract property name and value from equality expression."); + } + } + + /// + /// Extracts an inequality filter clause from a binary not-equal expression. + /// + /// The binary not-equal expression. + /// The list to add the extracted clause to. + private static void ExtractInequalityClause(BinaryExpression binaryExpr, List filterClauses) + { + // Note: Inequality is tracked but handled differently depending on the property + // For now, we log a warning that inequality filtering may not work as expected + string? propertyName = null; + object? value = null; + + if (binaryExpr.Left is MemberExpression leftMember) + { + propertyName = leftMember.Member.Name; + value = ExtractValue(binaryExpr.Right); + } + else if (binaryExpr.Right is MemberExpression rightMember) + { + propertyName = rightMember.Member.Name; + value = ExtractValue(binaryExpr.Left); + } + + if (propertyName != null && value != null) + { + // Add a marker for inequality - this will need special handling in conversion + // For now, we don't add it to filter clauses as Brave API doesn't support direct negation + throw new NotSupportedException($"Inequality operator (!=) is not directly supported for property '{propertyName}'. Use NOT operator instead: !(page.{propertyName} == value)."); + } + + throw new NotSupportedException("Unable to extract property name and value from inequality expression."); + } + + /// + /// Analyzes a NOT (negation) expression. + /// + /// The unary NOT expression. + /// The list to add extracted filter clauses to. + private static void AnalyzeNotExpression(UnaryExpression unaryExpr, List filterClauses) + { + // NOT expressions are complex for web search APIs + // We support simple cases like !(page.SafeSearch == "off") + if (unaryExpr.Operand is BinaryExpression binaryExpr && binaryExpr.NodeType == ExpressionType.Equal) + { + // This is !(property == value), which we can handle for some properties + throw new NotSupportedException("NOT operator (!) with equality is not directly supported. Most web search APIs don't support negative filtering."); + } + + throw new NotSupportedException("NOT operator (!) is only supported with simple equality expressions."); + } + + /// + /// Extracts a filter clause from a method call expression (e.g., Contains, StartsWith). + /// + /// The method call expression. + /// The list to add the extracted clause to. + private static void ExtractMethodCallClause(MethodCallExpression methodCall, List filterClauses) + { + if (methodCall.Method.Name == "Contains") + { + // Check if this is property.Contains(value) or array.Contains(property) + if (methodCall.Object is MemberExpression member) + { + // This is property.Contains(value) - e.g., page.ResultFilter.Contains("web") + var propertyName = member.Member.Name; + var value = ExtractValue(methodCall.Arguments[0]); + + if (value != null) + { + // For Contains, we'll map it to equality for certain properties + if (propertyName.Equals("ResultFilter", StringComparison.OrdinalIgnoreCase)) + { + filterClauses.Add(new EqualToFilterClause(propertyName, value)); + } + else if (propertyName.Equals("Title", StringComparison.OrdinalIgnoreCase)) + { + // For Title.Contains(), add the term to the search query itself + filterClauses.Add(new SearchQueryFilterClause(value.ToString() ?? string.Empty)); + } + else + { + throw new NotSupportedException($"Contains method is only supported for ResultFilter and Title properties, not '{propertyName}'."); + } + } + } + else if (methodCall.Object == null && methodCall.Arguments.Count == 2) + { + // This is array.Contains(property) - e.g., new[] { "US", "GB" }.Contains(page.Country) + // This pattern is not supported regardless of whether it's Enumerable.Contains (C# 13-) or MemoryExtensions.Contains (C# 14+) + // Both resolve to extension method calls with methodCall.Object == null + + // Provide detailed error message that covers both C# language versions + string errorMessage = "Collection Contains filters (e.g., array.Contains(page.Property)) are not supported by Brave Search API. " + + "Brave's API does not support OR logic across multiple values. "; + + if (IsMemoryExtensionsContains(methodCall)) + { + errorMessage += "Note: This occurs when using C# 14+ language features with span-based Contains methods (MemoryExtensions.Contains). "; + } + else + { + errorMessage += "Note: This occurs with standard LINQ extension methods (Enumerable.Contains). "; + } + + errorMessage += "Consider either: (1) performing multiple separate searches for each value, or " + + "(2) retrieving broader results and filtering on the client side."; + + throw new NotSupportedException(errorMessage); + } + else + { + throw new NotSupportedException("Unsupported Contains expression format."); + } + } + else + { + throw new NotSupportedException($"Method '{methodCall.Method.Name}' is not supported in Brave search filters. Only 'Contains' is supported."); + } + } + + /// + /// Extracts a constant value from an expression. + /// + /// The expression to extract the value from. + /// The extracted value, or null if extraction failed. + private static object? ExtractValue(Expression expression) + { + return expression switch + { + ConstantExpression constant => constant.Value, + MemberExpression member when member.Expression is ConstantExpression constantExpr => + member.Member switch + { + System.Reflection.FieldInfo field => field.GetValue(constantExpr.Value), + System.Reflection.PropertyInfo property => property.GetValue(constantExpr.Value), + _ => null + }, + _ => Expression.Lambda(expression).Compile().DynamicInvoke() + }; + } + + #endregion + + #region Private Methods private readonly ILogger _logger; private readonly HttpClient _httpClient; @@ -90,8 +521,19 @@ public async Task> GetSearchResultsAsync(string quer private static readonly ITextSearchStringMapper s_defaultStringMapper = new DefaultTextSearchStringMapper(); private static readonly ITextSearchResultMapper s_defaultResultMapper = new DefaultTextSearchResultMapper(); + // Constants for Brave API parameter names + private const string BraveParamCountry = "country"; + private const string BraveParamSearchLang = "search_lang"; + private const string BraveParamUiLang = "ui_lang"; + private const string BraveParamSafeSearch = "safesearch"; + private const string BraveParamTextDecorations = "text_decorations"; + private const string BraveParamSpellCheck = "spellcheck"; + private const string BraveParamResultFilter = "result_filter"; + private const string BraveParamUnits = "units"; + private const string BraveParamExtraSnippets = "extra_snippets"; + // See https://api-dashboard.search.brave.com/app/documentation/web-search/query#WebSearchAPIQueryParameters - private static readonly string[] s_queryParameters = ["country", "search_lang", "ui_lang", "safesearch", "text_decorations", "spellcheck", "result_filter", "units", "extra_snippets"]; + private static readonly string[] s_queryParameters = [BraveParamCountry, BraveParamSearchLang, BraveParamUiLang, BraveParamSafeSearch, BraveParamTextDecorations, BraveParamSpellCheck, BraveParamResultFilter, BraveParamUnits, BraveParamExtraSnippets]; private static readonly string[] s_safeSearch = ["off", "moderate", "strict"]; @@ -160,11 +602,36 @@ private async Task SendGetRequestAsync(string query, TextSe } /// - /// Return the search results as instances of . + /// Return the search results as instances of . + /// + /// Response containing the web pages matching the query. + /// Cancellation token + private async IAsyncEnumerable GetResultsAsObjectAsync(BraveSearchResponse? searchResponse, [EnumeratorCancellation] CancellationToken cancellationToken) + { + if (searchResponse?.Web?.Results is null) + { + yield break; + } + + foreach (var result in searchResponse.Web.Results) + { + yield return new BraveWebPage + { + Title = result.Title, + Url = string.IsNullOrWhiteSpace(result.Url) ? null : new Uri(result.Url), + Description = result.Description, + }; + + await Task.Yield(); + } + } + + /// + /// Return the search results as instances of . /// /// Response containing the web pages matching the query. /// Cancellation token - private async IAsyncEnumerable GetResultsAsWebPageAsync(BraveSearchResponse? searchResponse, [EnumeratorCancellation] CancellationToken cancellationToken) + private async IAsyncEnumerable GetResultsAsBraveWebPageAsync(BraveSearchResponse? searchResponse, [EnumeratorCancellation] CancellationToken cancellationToken) { if (searchResponse is null) { yield break; } @@ -172,7 +639,7 @@ private async IAsyncEnumerable GetResultsAsWebPageAsync(BraveSearchRespo { foreach (var webPage in webResults) { - yield return webPage; + yield return BraveWebPage.FromWebResult(webPage); await Task.Yield(); } } @@ -383,5 +850,42 @@ private static void CheckQueryValidation(string queryParam, object value) break; } } + + /// + /// Determines if a method call expression is a MemoryExtensions.Contains call (C# 14+ compatibility). + /// In C# 14+, array.Contains(property) may resolve to MemoryExtensions.Contains instead of Enumerable.Contains. + /// + /// The method call expression to check. + /// True if this is a MemoryExtensions.Contains call, false otherwise. + private static bool IsMemoryExtensionsContains(MethodCallExpression methodCall) + { + // Check if this is a static method call (Object is null) + if (methodCall.Object != null) + { + return false; + } + + // Check if it's MemoryExtensions.Contains + if (methodCall.Method.DeclaringType?.Name != "MemoryExtensions") + { + return false; + } + + // MemoryExtensions.Contains has 2-3 parameters: (ReadOnlySpan, T) or (ReadOnlySpan, T, IEqualityComparer) + if (methodCall.Arguments.Count < 2 || methodCall.Arguments.Count > 3) + { + return false; + } + + // For our text search scenarios, we don't support span comparers + if (methodCall.Arguments.Count == 3) + { + throw new NotSupportedException( + "MemoryExtensions.Contains with custom IEqualityComparer is not supported. " + + "Use simple array.Contains(property) expressions without custom comparers."); + } + + return true; + } #endregion } diff --git a/dotnet/src/Plugins/Plugins.Web/Brave/BraveWebPage.cs b/dotnet/src/Plugins/Plugins.Web/Brave/BraveWebPage.cs new file mode 100644 index 000000000000..c6938c7b0ef8 --- /dev/null +++ b/dotnet/src/Plugins/Plugins.Web/Brave/BraveWebPage.cs @@ -0,0 +1,145 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; + +namespace Microsoft.SemanticKernel.Plugins.Web.Brave; + +/// +/// Represents a type-safe web page result from Brave search for use with generic ITextSearch<TRecord> interface. +/// This class provides compile-time type safety and IntelliSense support for Brave search filtering. +/// +public sealed class BraveWebPage +{ + /// + /// Gets or sets the title of the web page. + /// + public string? Title { get; set; } + + /// + /// Gets or sets the URL of the web page. + /// + public Uri? Url { get; set; } + + /// + /// Gets or sets the description of the web page. + /// + public string? Description { get; set; } + + /// + /// Gets or sets the type of the search result. + /// + public string? Type { get; set; } + + /// + /// Gets or sets the age of the web search result. + /// + public string? Age { get; set; } + + /// + /// Gets or sets the page age timestamp. + /// + public DateTime? PageAge { get; set; } + + /// + /// Gets or sets the language of the web page. + /// + public string? Language { get; set; } + + /// + /// Gets or sets whether the web page is family friendly. + /// + public bool? FamilyFriendly { get; set; } + + /// + /// Gets or sets the country filter for search results. + /// Maps to Brave's 'country' parameter (e.g., "US", "GB", "CA"). + /// + public string? Country { get; set; } + + /// + /// Gets or sets the search language filter. + /// Maps to Brave's 'search_lang' parameter (e.g., "en", "es", "fr"). + /// + public string? SearchLang { get; set; } + + /// + /// Gets or sets the UI language filter. + /// Maps to Brave's 'ui_lang' parameter (e.g., "en-US", "en-GB"). + /// + public string? UiLang { get; set; } + + /// + /// Gets or sets the safe search filter. + /// Maps to Brave's 'safesearch' parameter ("off", "moderate", "strict"). + /// + public string? SafeSearch { get; set; } + + /// + /// Gets or sets whether text decorations are enabled. + /// Maps to Brave's 'text_decorations' parameter. + /// + public bool? TextDecorations { get; set; } + + /// + /// Gets or sets whether spell check is enabled. + /// Maps to Brave's 'spellcheck' parameter. + /// + public bool? SpellCheck { get; set; } + + /// + /// Gets or sets the result filter for search types. + /// Maps to Brave's 'result_filter' parameter (e.g., "web", "news", "videos"). + /// + public string? ResultFilter { get; set; } + + /// + /// Gets or sets the units system for measurements. + /// Maps to Brave's 'units' parameter ("metric" or "imperial"). + /// + public string? Units { get; set; } + + /// + /// Gets or sets whether extra snippets are included. + /// Maps to Brave's 'extra_snippets' parameter. + /// + public bool? ExtraSnippets { get; set; } + + /// + /// Initializes a new instance of the class. + /// + public BraveWebPage() + { + } + + /// + /// Initializes a new instance of the class with specified values. + /// + /// The title of the web page. + /// The URL of the web page. + /// The description of the web page. + /// The type of the search result. + public BraveWebPage(string? title, Uri? url, string? description, string? type = null) + { + this.Title = title; + this.Url = url; + this.Description = description; + this.Type = type; + } + + /// + /// Creates a BraveWebPage from a BraveWebResult. + /// + /// The web result to convert. + /// A new BraveWebPage instance. + internal static BraveWebPage FromWebResult(BraveWebResult result) + { + Uri? url = string.IsNullOrWhiteSpace(result.Url) ? null : new Uri(result.Url); + return new BraveWebPage(result.Title, url, result.Description, result.Type) + { + Age = result.Age, + PageAge = result.PageAge, + Language = result.Language, + FamilyFriendly = result.FamilyFriendly + }; + } +} diff --git a/dotnet/src/Plugins/Plugins.Web/FilterClauses/SearchQueryFilterClause.cs b/dotnet/src/Plugins/Plugins.Web/FilterClauses/SearchQueryFilterClause.cs new file mode 100644 index 000000000000..9909da9579e6 --- /dev/null +++ b/dotnet/src/Plugins/Plugins.Web/FilterClauses/SearchQueryFilterClause.cs @@ -0,0 +1,38 @@ +// Copyright (c) Microsoft. All rights reserved. + +using Microsoft.Extensions.VectorData; + +namespace Microsoft.SemanticKernel.Plugins.Web; + +/// +/// Represents a filter clause that adds terms to the search query itself for text search engines. +/// +/// +/// This filter clause is used when the underlying search service should add the specified +/// terms to the search query to help find matching results, rather than filtering results +/// after they are returned. +/// +/// Primary use case: Supporting Title.Contains("value") LINQ expressions for search engines +/// that don't have field-specific operators (e.g., Brave, Tavily). The implementation extracts +/// the search term and appends it to the base query for enhanced relevance. +/// +/// Example: Title.Contains("AI") → SearchQueryFilterClause("AI") → query + " AI" +/// +/// See ADR-TextSearch-Contains-Support.md for architectural context and cross-engine comparison. +/// +internal sealed class SearchQueryFilterClause : FilterClause +{ + /// + /// Initializes a new instance of the class. + /// + /// The term to add to the search query. + public SearchQueryFilterClause(string searchTerm) + { + this.SearchTerm = searchTerm; + } + + /// + /// Gets the search term to add to the query. + /// + public string SearchTerm { get; private set; } +} diff --git a/dotnet/src/Plugins/Plugins.Web/Google/GoogleTextSearch.cs b/dotnet/src/Plugins/Plugins.Web/Google/GoogleTextSearch.cs index c4165a2edadc..c450e4f0d4e5 100644 --- a/dotnet/src/Plugins/Plugins.Web/Google/GoogleTextSearch.cs +++ b/dotnet/src/Plugins/Plugins.Web/Google/GoogleTextSearch.cs @@ -2,6 +2,8 @@ using System; using System.Collections.Generic; +using System.Linq; +using System.Linq.Expressions; using System.Runtime.CompilerServices; using System.Threading; using System.Threading.Tasks; @@ -17,7 +19,9 @@ namespace Microsoft.SemanticKernel.Plugins.Web.Google; /// /// A Google Text Search implementation that can be used to perform searches using the Google Web Search API. /// -public sealed class GoogleTextSearch : ITextSearch, IDisposable +#pragma warning disable CS0618 // ITextSearch is obsolete - this class provides backward compatibility +public sealed class GoogleTextSearch : ITextSearch, ITextSearch, IDisposable +#pragma warning restore CS0618 { /// /// Initializes a new instance of the class. @@ -87,15 +91,343 @@ public async Task> SearchAsync(string query, TextSea return new KernelSearchResults(this.GetResultsAsStringAsync(searchResponse, cancellationToken), totalCount, GetResultsMetadata(searchResponse)); } + #region ITextSearch Implementation + /// - public void Dispose() + public async Task> GetSearchResultsAsync(string query, TextSearchOptions? searchOptions = null, CancellationToken cancellationToken = default) { - this._search.Dispose(); + var legacyOptions = ConvertToLegacyOptions(searchOptions); + var searchResponse = await this.ExecuteSearchAsync(query, legacyOptions, cancellationToken).ConfigureAwait(false); + + long? totalCount = searchOptions?.IncludeTotalCount == true ? long.Parse(searchResponse.SearchInformation.TotalResults) : null; + + return new KernelSearchResults(this.GetResultsAsGoogleWebPageAsync(searchResponse, cancellationToken), totalCount, GetResultsMetadata(searchResponse)); } - #region private + /// + public async Task> GetTextSearchResultsAsync(string query, TextSearchOptions? searchOptions = null, CancellationToken cancellationToken = default) + { + var legacyOptions = ConvertToLegacyOptions(searchOptions); + var searchResponse = await this.ExecuteSearchAsync(query, legacyOptions, cancellationToken).ConfigureAwait(false); - private const int MaxCount = 10; + long? totalCount = searchOptions?.IncludeTotalCount == true ? long.Parse(searchResponse.SearchInformation.TotalResults) : null; + + return new KernelSearchResults(this.GetResultsAsTextSearchResultAsync(searchResponse, cancellationToken), totalCount, GetResultsMetadata(searchResponse)); + } + + /// + public async Task> SearchAsync(string query, TextSearchOptions? searchOptions = null, CancellationToken cancellationToken = default) + { + var legacyOptions = ConvertToLegacyOptions(searchOptions); + var searchResponse = await this.ExecuteSearchAsync(query, legacyOptions, cancellationToken).ConfigureAwait(false); + + long? totalCount = searchOptions?.IncludeTotalCount == true ? long.Parse(searchResponse.SearchInformation.TotalResults) : null; + + return new KernelSearchResults(this.GetResultsAsStringAsync(searchResponse, cancellationToken), totalCount, GetResultsMetadata(searchResponse)); + } + + /// + /// Converts generic TextSearchOptions with LINQ filtering to legacy TextSearchOptions. + /// Attempts to translate simple LINQ expressions to Google API filters where possible. + /// + /// The generic search options with LINQ filtering. + /// Legacy TextSearchOptions with equivalent filtering. + private static TextSearchOptions ConvertToLegacyOptions(TextSearchOptions? genericOptions) + { + if (genericOptions == null) + { + return new TextSearchOptions(); + } + + return new TextSearchOptions + { + Top = genericOptions.Top, + Skip = genericOptions.Skip, + IncludeTotalCount = genericOptions.IncludeTotalCount, + Filter = genericOptions.Filter != null ? ConvertLinqExpressionToGoogleFilter(genericOptions.Filter) : null + }; + } + + /// + /// Converts a LINQ expression to a TextSearchFilter compatible with Google Custom Search API. + /// Supports property equality expressions, string Contains operations, NOT operations (inequality and negation), + /// and compound AND expressions that map to Google's filter capabilities. + /// + /// The LINQ expression to convert. + /// A TextSearchFilter with equivalent filtering. + /// Thrown when the expression cannot be converted to Google filters. + private static TextSearchFilter ConvertLinqExpressionToGoogleFilter(Expression> linqExpression) + { + // Handle compound AND expressions: expr1 && expr2 + if (linqExpression.Body is BinaryExpression andExpr && andExpr.NodeType == ExpressionType.AndAlso) + { + var filter = new TextSearchFilter(); + CollectAndCombineFilters(andExpr, filter); + return filter; + } + + // Handle simple expressions using the shared processing logic + var textSearchFilter = new TextSearchFilter(); + if (TryProcessSingleExpression(linqExpression.Body, textSearchFilter)) + { + return textSearchFilter; + } + + // Generate helpful error message with supported patterns + var supportedProperties = s_queryParameters.Select(p => + MapGoogleFilterToProperty(p)).Where(p => p != null).Distinct(); + + throw new NotSupportedException( + $"LINQ expression '{linqExpression}' cannot be converted to Google API filters. " + + $"Supported patterns: {string.Join(", ", s_supportedPatterns)}. " + + $"Supported properties: {string.Join(", ", supportedProperties)}."); + } + + /// + /// Recursively collects and combines filters from compound AND expressions. + /// + /// The expression to process. + /// The filter to accumulate results into. + private static void CollectAndCombineFilters(Expression expression, TextSearchFilter filter) + { + if (expression is BinaryExpression binaryExpr && binaryExpr.NodeType == ExpressionType.AndAlso) + { + // Recursively process both sides of the AND + CollectAndCombineFilters(binaryExpr.Left, filter); + CollectAndCombineFilters(binaryExpr.Right, filter); + } + else + { + // Process individual expression using shared logic + TryProcessSingleExpression(expression, filter); + } + } + + /// + /// Shared logic to process a single LINQ expression and add appropriate filters. + /// Consolidates duplicate code between ConvertLinqExpressionToGoogleFilter and CollectAndCombineFilters. + /// + /// The expression to process. + /// The filter to add results to. + /// True if the expression was successfully processed, false otherwise. + private static bool TryProcessSingleExpression(Expression expression, TextSearchFilter filter) + { + // Handle equality: record.PropertyName == "value" + if (expression is BinaryExpression equalExpr && equalExpr.NodeType == ExpressionType.Equal) + { + return TryProcessEqualityExpression(equalExpr, filter); + } + + // Handle inequality (NOT): record.PropertyName != "value" + if (expression is BinaryExpression notEqualExpr && notEqualExpr.NodeType == ExpressionType.NotEqual) + { + return TryProcessInequalityExpression(notEqualExpr, filter); + } + + // Handle Contains method calls + if (expression is MethodCallExpression methodCall && methodCall.Method.Name == "Contains") + { + // String.Contains (instance method) - supported for substring search + if (methodCall.Method.DeclaringType == typeof(string)) + { + return TryProcessContainsExpression(methodCall, filter); + } + + // Collection Contains (static methods) - NOT supported due to Google API limitations + // This handles both Enumerable.Contains (C# 13-) and MemoryExtensions.Contains (C# 14+) + // User's C# language version determines which method is resolved, but both are unsupported + if (methodCall.Object == null) // Static method + { + // Enumerable.Contains or MemoryExtensions.Contains + if (methodCall.Method.DeclaringType == typeof(Enumerable) || + (methodCall.Method.DeclaringType == typeof(MemoryExtensions) && IsMemoryExtensionsContains(methodCall))) + { + throw new NotSupportedException( + "Collection Contains filters (e.g., array.Contains(page.Property)) are not supported by Google Custom Search API. " + + "Google's search operators do not support OR logic across multiple values. " + + "Consider either: (1) performing multiple separate searches for each value, or " + + "(2) retrieving broader results and filtering on the client side."); + } + } + } + + // Handle NOT expressions: !record.PropertyName.Contains("value") + if (expression is UnaryExpression unaryExpr && unaryExpr.NodeType == ExpressionType.Not) + { + return TryProcessNotExpression(unaryExpr, filter); + } + + return false; + } + + /// + /// Checks if a method call expression is MemoryExtensions.Contains. + /// This handles C# 14's "first-class spans" feature where collection.Contains(item) resolves to + /// MemoryExtensions.Contains instead of Enumerable.Contains. + /// + private static bool IsMemoryExtensionsContains(MethodCallExpression methodExpr) + { + // MemoryExtensions.Contains has 2-3 parameters (source, value, optional comparer) + // We only support the case without a comparer (or with null comparer) + return methodExpr.Method.Name == nameof(MemoryExtensions.Contains) && + methodExpr.Arguments.Count >= 2 && + methodExpr.Arguments.Count <= 3 && + (methodExpr.Arguments.Count == 2 || + (methodExpr.Arguments.Count == 3 && methodExpr.Arguments[2] is ConstantExpression { Value: null })); + } + + /// + /// Processes equality expressions: record.PropertyName == "value" + /// + private static bool TryProcessEqualityExpression(BinaryExpression equalExpr, TextSearchFilter filter) + { + if (equalExpr.Left is MemberExpression memberExpr && equalExpr.Right is ConstantExpression constExpr) + { + string propertyName = memberExpr.Member.Name; + object? value = constExpr.Value; + string? googleFilterName = MapPropertyToGoogleFilter(propertyName); + if (googleFilterName != null && value != null) + { + filter.Equality(googleFilterName, value); + return true; + } + } + return false; + } + + /// + /// Processes inequality expressions: record.PropertyName != "value" + /// + private static bool TryProcessInequalityExpression(BinaryExpression notEqualExpr, TextSearchFilter filter) + { + if (notEqualExpr.Left is MemberExpression memberExpr && notEqualExpr.Right is ConstantExpression constExpr) + { + string propertyName = memberExpr.Member.Name; + object? value = constExpr.Value; + // Map to excludeTerms for text fields + if (propertyName.ToUpperInvariant() is "TITLE" or "SNIPPET" && value != null) + { + filter.Equality("excludeTerms", value); + return true; + } + } + return false; + } + + /// + /// Processes Contains expressions: record.PropertyName.Contains("value") + /// + private static bool TryProcessContainsExpression(MethodCallExpression methodCall, TextSearchFilter filter) + { + if (methodCall.Object is MemberExpression memberExpr && + methodCall.Arguments.Count == 1 && + methodCall.Arguments[0] is ConstantExpression constExpr) + { + string propertyName = memberExpr.Member.Name; + object? value = constExpr.Value; + string? googleFilterName = MapPropertyToGoogleFilter(propertyName); + if (googleFilterName != null && value != null) + { + // For Contains operations on text fields, use exactTerms or orTerms + if (googleFilterName == "exactTerms") + { + filter.Equality("orTerms", value); // More flexible than exactTerms + } + else + { + filter.Equality(googleFilterName, value); + } + return true; + } + } + return false; + } + + /// + /// Processes NOT expressions: !record.PropertyName.Contains("value") + /// + private static bool TryProcessNotExpression(UnaryExpression unaryExpr, TextSearchFilter filter) + { + if (unaryExpr.Operand is MethodCallExpression notMethodCall && + notMethodCall.Method.Name == "Contains" && + notMethodCall.Method.DeclaringType == typeof(string)) + { + if (notMethodCall.Object is MemberExpression memberExpr && + notMethodCall.Arguments.Count == 1 && + notMethodCall.Arguments[0] is ConstantExpression constExpr) + { + string propertyName = memberExpr.Member.Name; + object? value = constExpr.Value; + if (propertyName.ToUpperInvariant() is "TITLE" or "SNIPPET" && value != null) + { + filter.Equality("excludeTerms", value); + return true; + } + } + } + return false; + } + + /// + /// Maps GoogleWebPage property names to Google Custom Search API filter field names. + /// + /// The GoogleWebPage property name. + /// The corresponding Google API filter name, or null if not mappable. + private static string? MapPropertyToGoogleFilter(string propertyName) + { + return propertyName.ToUpperInvariant() switch + { + // Map GoogleWebPage properties to Google API equivalents + "LINK" => "siteSearch", // Maps to site search + "DISPLAYLINK" => "siteSearch", // Maps to site search + "TITLE" => "exactTerms", // Exact title match + "SNIPPET" => "exactTerms", // Exact content match + + // Direct API parameters mapped from GoogleWebPage metadata properties + "FILEFORMAT" => "fileType", // File type/extension filtering + "MIME" => "filter", // MIME type filtering + + // Locale/Language parameters (if we extend GoogleWebPage) + "HL" => "hl", // Interface language + "GL" => "gl", // Geolocation + "CR" => "cr", // Country restrict + "LR" => "lr", // Language restrict + + _ => null // Property not mappable to Google filters + }; + } + + /// + /// Maps Google Custom Search API filter field names back to example GoogleWebPage property names. + /// Used for generating helpful error messages. + /// + /// The Google API filter name. + /// An example property name, or null if not mappable. + private static string? MapGoogleFilterToProperty(string googleFilterName) + { + return googleFilterName switch + { + "siteSearch" => "DisplayLink", + "exactTerms" => "Title", + "orTerms" => "Title", + "excludeTerms" => "Title", + "fileType" => "FileFormat", + "filter" => "Mime", + "hl" => "HL", + "gl" => "GL", + "cr" => "CR", + "lr" => "LR", + _ => null + }; + } + + #endregion + + /// + public void Dispose() + { + this._search.Dispose(); + } private readonly ILogger _logger; private readonly CustomSearchAPIService _search; @@ -106,8 +438,19 @@ public void Dispose() private static readonly ITextSearchStringMapper s_defaultStringMapper = new DefaultTextSearchStringMapper(); private static readonly ITextSearchResultMapper s_defaultResultMapper = new DefaultTextSearchResultMapper(); + private const int MaxCount = 10; + // See https://developers.google.com/custom-search/v1/reference/rest/v1/cse/list - private static readonly string[] s_queryParameters = ["cr", "dateRestrict", "exactTerms", "excludeTerms", "filter", "gl", "hl", "linkSite", "lr", "orTerms", "rights", "siteSearch"]; + private static readonly string[] s_queryParameters = ["cr", "dateRestrict", "exactTerms", "excludeTerms", "fileType", "filter", "gl", "hl", "linkSite", "lr", "orTerms", "rights", "siteSearch"]; + + // Performance optimization: Static error message arrays to avoid allocations in error paths + private static readonly string[] s_supportedPatterns = [ + "page.Property == \"value\" (exact match)", + "page.Property != \"value\" (exclude)", + "page.Property.Contains(\"text\") (partial match)", + "!page.Property.Contains(\"text\") (exclude partial)", + "page.Prop1 == \"val1\" && page.Prop2.Contains(\"val2\") (compound AND)" + ]; private delegate void SetSearchProperty(CseResource.ListRequest search, string value); @@ -116,6 +459,7 @@ public void Dispose() { "DATERESTRICT", (search, value) => search.DateRestrict = value }, { "EXACTTERMS", (search, value) => search.ExactTerms = value }, { "EXCLUDETERMS", (search, value) => search.ExcludeTerms = value }, + { "FILETYPE", (search, value) => search.FileType = value }, { "FILTER", (search, value) => search.Filter = value }, { "GL", (search, value) => search.Gl = value }, { "HL", (search, value) => search.Hl = value }, @@ -139,7 +483,7 @@ public void Dispose() var count = searchOptions.Top; var offset = searchOptions.Skip; - if (count is <= 0 or > MaxCount) + if (count <= 0 || count > MaxCount) { throw new ArgumentOutOfRangeException(nameof(searchOptions), count, $"{nameof(searchOptions)}.Count value must be must be greater than 0 and less than or equals 10."); } @@ -233,6 +577,25 @@ private async IAsyncEnumerable GetResultsAsStringAsync(global::Google.Ap } } + /// + /// Return the search results as instances of . + /// + /// Google search response + /// Cancellation token + private async IAsyncEnumerable GetResultsAsGoogleWebPageAsync(global::Google.Apis.CustomSearchAPI.v1.Data.Search searchResponse, [EnumeratorCancellation] CancellationToken cancellationToken) + { + if (searchResponse is null || searchResponse.Items is null) + { + yield break; + } + + foreach (var item in searchResponse.Items) + { + yield return ConvertToGoogleWebPage(item); + await Task.Yield(); + } + } + /// /// Return the search results as instances of . /// @@ -264,6 +627,29 @@ private async IAsyncEnumerable GetResultsAsStringAsync(global::Google.Ap }; } + /// + /// Converts a Google CustomSearchAPI Result to a GoogleWebPage instance. + /// + /// The Google search result to convert. + /// A GoogleWebPage with mapped properties. + private static GoogleWebPage ConvertToGoogleWebPage(global::Google.Apis.CustomSearchAPI.v1.Data.Result googleResult) + { + return new GoogleWebPage + { + Title = googleResult.Title, + Link = googleResult.Link, + Snippet = googleResult.Snippet, + DisplayLink = googleResult.DisplayLink, + FormattedUrl = googleResult.FormattedUrl, + HtmlFormattedUrl = googleResult.HtmlFormattedUrl, + HtmlSnippet = googleResult.HtmlSnippet, + HtmlTitle = googleResult.HtmlTitle, + Mime = googleResult.Mime, + FileFormat = googleResult.FileFormat, + Labels = googleResult.Labels?.Select(l => l.Name).ToArray() + }; + } + /// /// Default implementation which maps from a to a /// @@ -297,5 +683,4 @@ public TextSearchResult MapFromResultToTextSearchResult(object result) return new TextSearchResult(googleResult.Snippet) { Name = googleResult.Title, Link = googleResult.Link }; } } - #endregion } diff --git a/dotnet/src/Plugins/Plugins.Web/Google/GoogleWebPage.cs b/dotnet/src/Plugins/Plugins.Web/Google/GoogleWebPage.cs new file mode 100644 index 000000000000..8eab2153d27b --- /dev/null +++ b/dotnet/src/Plugins/Plugins.Web/Google/GoogleWebPage.cs @@ -0,0 +1,103 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System.Collections.Generic; +using System.Text.Json.Serialization; + +namespace Microsoft.SemanticKernel.Plugins.Web.Google; + +/// +/// Defines a webpage result from Google Custom Search API. +/// +public sealed class GoogleWebPage +{ + /// + /// Only allow creation within this package. + /// + [JsonConstructorAttribute] + internal GoogleWebPage() + { + } + + /// + /// Gets or sets the title of the webpage. + /// + /// + /// Use this title along with Link to create a hyperlink that when clicked takes the user to the webpage. + /// + [JsonPropertyName("title")] + public string? Title { get; set; } + + /// + /// Gets or sets the URL to the webpage. + /// + /// + /// Use this URL along with Title to create a hyperlink that when clicked takes the user to the webpage. + /// + [JsonPropertyName("link")] +#pragma warning disable CA1056 // URI-like properties should not be strings + public string? Link { get; set; } +#pragma warning restore CA1056 // URI-like properties should not be strings + + /// + /// Gets or sets a snippet of text from the webpage that describes its contents. + /// + [JsonPropertyName("snippet")] + public string? Snippet { get; set; } + + /// + /// Gets or sets the formatted URL display string. + /// + /// + /// The URL is meant for display purposes only and may not be well formed. + /// + [JsonPropertyName("displayLink")] +#pragma warning disable CA1056 // URI-like properties should not be strings + public string? DisplayLink { get; set; } +#pragma warning restore CA1056 // URI-like properties should not be strings + + /// + /// Gets or sets the MIME type of the result. + /// + [JsonPropertyName("mime")] + public string? Mime { get; set; } + + /// + /// Gets or sets the file format of the result. + /// + [JsonPropertyName("fileFormat")] + public string? FileFormat { get; set; } + + /// + /// Gets or sets the HTML title of the webpage. + /// + [JsonPropertyName("htmlTitle")] + public string? HtmlTitle { get; set; } + + /// + /// Gets or sets the HTML snippet of the webpage. + /// + [JsonPropertyName("htmlSnippet")] + public string? HtmlSnippet { get; set; } + + /// + /// Gets or sets the formatted URL of the webpage. + /// + [JsonPropertyName("formattedUrl")] +#pragma warning disable CA1056 // URI-like properties should not be strings + public string? FormattedUrl { get; set; } +#pragma warning restore CA1056 // URI-like properties should not be strings + + /// + /// Gets or sets the HTML-formatted URL of the webpage. + /// + [JsonPropertyName("htmlFormattedUrl")] +#pragma warning disable CA1056 // URI-like properties should not be strings + public string? HtmlFormattedUrl { get; set; } +#pragma warning restore CA1056 // URI-like properties should not be strings + + /// + /// Gets or sets labels associated with the webpage. + /// + [JsonPropertyName("labels")] + public IReadOnlyList? Labels { get; set; } +} diff --git a/dotnet/src/Plugins/Plugins.Web/Tavily/TavilyTextSearch.cs b/dotnet/src/Plugins/Plugins.Web/Tavily/TavilyTextSearch.cs index 4e01d0ffb88b..ab06f08cb9ad 100644 --- a/dotnet/src/Plugins/Plugins.Web/Tavily/TavilyTextSearch.cs +++ b/dotnet/src/Plugins/Plugins.Web/Tavily/TavilyTextSearch.cs @@ -2,6 +2,7 @@ using System; using System.Collections.Generic; +using System.Linq.Expressions; using System.Net.Http; using System.Runtime.CompilerServices; using System.Text; @@ -20,7 +21,9 @@ namespace Microsoft.SemanticKernel.Plugins.Web.Tavily; /// /// A Tavily Text Search implementation that can be used to perform searches using the Tavily Web Search API. /// -public sealed class TavilyTextSearch : ITextSearch +#pragma warning disable CS0618 // ITextSearch is obsolete - this class provides backward compatibility +public sealed class TavilyTextSearch : ITextSearch, ITextSearch +#pragma warning restore CS0618 { /// /// Create an instance of the with API key authentication. @@ -75,7 +78,431 @@ public async Task> GetSearchResultsAsync(string quer return new KernelSearchResults(this.GetSearchResultsAsync(searchResponse, cancellationToken), totalCount, GetResultsMetadata(searchResponse)); } - #region private + #region Generic ITextSearch Implementation + + /// + async Task> ITextSearch.SearchAsync(string query, TextSearchOptions? searchOptions, CancellationToken cancellationToken) + { + var (modifiedQuery, legacyOptions) = this.ConvertToLegacyOptionsWithQuery(query, searchOptions); + TavilySearchResponse? searchResponse = await this.ExecuteSearchAsync(modifiedQuery, legacyOptions, cancellationToken).ConfigureAwait(false); + + long? totalCount = null; + + return new KernelSearchResults(this.GetResultsAsStringAsync(searchResponse, cancellationToken), totalCount, GetResultsMetadata(searchResponse)); + } + + /// + async Task> ITextSearch.GetTextSearchResultsAsync(string query, TextSearchOptions? searchOptions, CancellationToken cancellationToken) + { + var (modifiedQuery, legacyOptions) = this.ConvertToLegacyOptionsWithQuery(query, searchOptions); + TavilySearchResponse? searchResponse = await this.ExecuteSearchAsync(modifiedQuery, legacyOptions, cancellationToken).ConfigureAwait(false); + + long? totalCount = null; + + return new KernelSearchResults(this.GetResultsAsTextSearchResultAsync(searchResponse, cancellationToken), totalCount, GetResultsMetadata(searchResponse)); + } + + /// + async Task> ITextSearch.GetSearchResultsAsync(string query, TextSearchOptions? searchOptions, CancellationToken cancellationToken) + { + var (modifiedQuery, legacyOptions) = this.ConvertToLegacyOptionsWithQuery(query, searchOptions); + TavilySearchResponse? searchResponse = await this.ExecuteSearchAsync(modifiedQuery, legacyOptions, cancellationToken).ConfigureAwait(false); + + long? totalCount = null; + + return new KernelSearchResults(this.GetResultsAsWebPageAsync(searchResponse, cancellationToken), totalCount, GetResultsMetadata(searchResponse)); + } + + #endregion + + #region LINQ-to-Tavily Conversion Logic + + /// + /// Converts generic TextSearchOptions with LINQ filtering to legacy TextSearchOptions and extracts additional search terms. + /// + /// The original search query. + /// The generic search options with LINQ filter. + /// A tuple containing the modified query and legacy TextSearchOptions with converted filters. + private (string modifiedQuery, TextSearchOptions legacyOptions) ConvertToLegacyOptionsWithQuery(string query, TextSearchOptions? options) + { + var legacyOptions = this.ConvertToLegacyOptions(options); + + if (options?.Filter != null) + { + // Extract search terms from the LINQ expression + var additionalSearchTerms = ExtractSearchTermsFromLinqExpression(options.Filter); + if (additionalSearchTerms.Count > 0) + { + // Append additional search terms to the original query + var modifiedQuery = $"{query} {string.Join(" ", additionalSearchTerms)}".Trim(); + return (modifiedQuery, legacyOptions); + } + } + + return (query, legacyOptions); + } + + /// + /// Converts generic TextSearchOptions with LINQ filtering to legacy TextSearchOptions. + /// + /// The generic search options with LINQ filter. + /// Legacy TextSearchOptions with converted filters. + private TextSearchOptions ConvertToLegacyOptions(TextSearchOptions? options) + { + if (options == null) + { + return new TextSearchOptions(); + } + + var legacyOptions = new TextSearchOptions + { + Top = options.Top, + Skip = options.Skip, + IncludeTotalCount = options.IncludeTotalCount + }; + + // Convert LINQ expression to TextSearchFilter if present + if (options.Filter != null) + { + try + { + var convertedFilter = ConvertLinqExpressionToTavilyFilter(options.Filter); + legacyOptions = new TextSearchOptions + { + Top = options.Top, + Skip = options.Skip, + IncludeTotalCount = options.IncludeTotalCount, + Filter = convertedFilter + }; + } + catch (NotSupportedException) + { + // All unsupported LINQ patterns should fail explicitly to provide clear developer feedback + // This helps developers understand which patterns work with the Tavily API + throw; + } + } + + return legacyOptions; + } + + /// + /// Extracts search terms that should be added to the search query from a LINQ expression. + /// + /// The LINQ expression to analyze. + /// A list of search terms to add to the query. + private static List ExtractSearchTermsFromLinqExpression(Expression> linqExpression) + { + var searchTerms = new List(); + var filterClauses = new List(); + + // Analyze the LINQ expression to get all filter clauses + AnalyzeExpression(linqExpression.Body, filterClauses); + + // Extract search terms from SearchQueryFilterClause instances + foreach (var clause in filterClauses) + { + if (clause is SearchQueryFilterClause searchQueryClause) + { + searchTerms.Add(searchQueryClause.SearchTerm); + } + } + + return searchTerms; + } + + /// + /// Converts a LINQ expression to Tavily-compatible TextSearchFilter. + /// + /// The LINQ expression to convert. + /// A TextSearchFilter with Tavily-compatible filter clauses. + private static TextSearchFilter ConvertLinqExpressionToTavilyFilter(Expression> linqExpression) + { + var filter = new TextSearchFilter(); + var filterClauses = new List(); + + // Analyze the LINQ expression and convert to filter clauses + AnalyzeExpression(linqExpression.Body, filterClauses); + + // Validate and add clauses that are supported by Tavily + foreach (var clause in filterClauses) + { + if (clause is EqualToFilterClause equalityClause) + { + var mappedFieldName = MapPropertyToTavilyFilter(equalityClause.FieldName); + if (mappedFieldName != null) + { + filter.Equality(mappedFieldName, equalityClause.Value); + } + else + { + throw new NotSupportedException( + $"Property '{equalityClause.FieldName}' cannot be mapped to Tavily API filters. " + + $"Supported properties: {string.Join(", ", s_validFieldNames)}. " + + "Example: page => page.Topic == \"general\" && page.TimeRange == \"week\""); + } + } + else if (clause is SearchQueryFilterClause) + { + // SearchQueryFilterClause is handled at the query level, not the filter level + // Skip it here as it's processed by ConvertToLegacyOptionsWithQuery + continue; + } + } + + return filter; + } + + /// + /// Maps TavilyWebPage property names to Tavily API filter parameter names. + /// + /// The property name from TavilyWebPage. + /// The corresponding Tavily API parameter name, or null if not mappable. + private static string? MapPropertyToTavilyFilter(string propertyName) => + propertyName.ToUpperInvariant() switch + { + "TOPIC" => Topic, + "TIMERANGE" => TimeRange, + "DAYS" => Days, + "INCLUDEDOMAIN" => IncludeDomain, + "EXCLUDEDOMAIN" => ExcludeDomain, + _ => null // Property not mappable to Tavily filters + }; + + // TODO: Consider extracting LINQ expression analysis logic to a shared utility class + // to reduce duplication across text search connectors (Brave, Tavily, etc.). + // See code review for details. + /// + /// Analyzes a LINQ expression and extracts filter clauses. + /// + /// The expression to analyze. + /// The list to add extracted filter clauses to. + private static void AnalyzeExpression(Expression expression, List filterClauses) + { + switch (expression) + { + case BinaryExpression binaryExpr: + if (binaryExpr.NodeType == ExpressionType.AndAlso) + { + // Handle AND expressions by recursively analyzing both sides + AnalyzeExpression(binaryExpr.Left, filterClauses); + AnalyzeExpression(binaryExpr.Right, filterClauses); + } + else if (binaryExpr.NodeType == ExpressionType.OrElse) + { + // Handle OR expressions by recursively analyzing both sides + // Note: OR results in multiple filter values for the same property (especially for domains) + AnalyzeExpression(binaryExpr.Left, filterClauses); + AnalyzeExpression(binaryExpr.Right, filterClauses); + } + else if (binaryExpr.NodeType == ExpressionType.Equal) + { + // Handle equality expressions + ExtractEqualityClause(binaryExpr, filterClauses); + } + else if (binaryExpr.NodeType == ExpressionType.NotEqual) + { + // Handle inequality expressions (property != value) + // This is supported as a negation pattern + ExtractInequalityClause(binaryExpr, filterClauses); + } + else + { + throw new NotSupportedException($"Binary expression type '{binaryExpr.NodeType}' is not supported. Supported operators: AndAlso (&&), OrElse (||), Equal (==), NotEqual (!=)."); + } + break; + + case UnaryExpression unaryExpr when unaryExpr.NodeType == ExpressionType.Not: + // Handle NOT expressions (negation) + AnalyzeNotExpression(unaryExpr, filterClauses); + break; + + case MethodCallExpression methodCall: + // Handle method calls like Contains, StartsWith, etc. + ExtractMethodCallClause(methodCall, filterClauses); + break; + + default: + throw new NotSupportedException($"Expression type '{expression.NodeType}' is not supported in Tavily search filters."); + } + } + + /// + /// Extracts an equality filter clause from a binary equality expression. + /// + /// The binary equality expression. + /// The list to add the extracted clause to. + private static void ExtractEqualityClause(BinaryExpression binaryExpr, List filterClauses) + { + string? propertyName = null; + object? value = null; + + // Determine which side is the property and which is the value + if (binaryExpr.Left is MemberExpression leftMember) + { + propertyName = leftMember.Member.Name; + value = ExtractValue(binaryExpr.Right); + } + else if (binaryExpr.Right is MemberExpression rightMember) + { + propertyName = rightMember.Member.Name; + value = ExtractValue(binaryExpr.Left); + } + + if (propertyName != null && value != null) + { + filterClauses.Add(new EqualToFilterClause(propertyName, value)); + } + else + { + throw new NotSupportedException("Unable to extract property name and value from equality expression."); + } + } + + /// + /// Extracts an inequality filter clause from a binary not-equal expression. + /// + /// The binary not-equal expression. + /// The list to add the extracted clause to. + private static void ExtractInequalityClause(BinaryExpression binaryExpr, List filterClauses) + { + // Note: Inequality is tracked but handled differently depending on the property + // For now, we log a warning that inequality filtering may not work as expected + string? propertyName = null; + object? value = null; + + if (binaryExpr.Left is MemberExpression leftMember) + { + propertyName = leftMember.Member.Name; + value = ExtractValue(binaryExpr.Right); + } + else if (binaryExpr.Right is MemberExpression rightMember) + { + propertyName = rightMember.Member.Name; + value = ExtractValue(binaryExpr.Left); + } + + if (propertyName != null && value != null) + { + // Add a marker for inequality - this will need special handling in conversion + // For now, we don't add it to filter clauses as Tavily API doesn't support direct negation + throw new NotSupportedException($"Inequality operator (!=) is not directly supported for property '{propertyName}'. Use NOT operator instead: !(page.{propertyName} == value)."); + } + + throw new NotSupportedException("Unable to extract property name and value from inequality expression."); + } + + /// + /// Analyzes a NOT (negation) expression. + /// + /// The unary NOT expression. + /// The list to add extracted filter clauses to. + private static void AnalyzeNotExpression(UnaryExpression unaryExpr, List filterClauses) + { + // NOT expressions are complex for web search APIs + // We support simple cases like !(page.Topic == "general") + if (unaryExpr.Operand is BinaryExpression binaryExpr && binaryExpr.NodeType == ExpressionType.Equal) + { + // This is !(property == value), which we can handle for some properties + throw new NotSupportedException("NOT operator (!) with equality is not directly supported. Most web search APIs don't support negative filtering."); + } + + throw new NotSupportedException("NOT operator (!) is only supported with simple equality expressions."); + } + + /// + /// Extracts a filter clause from a method call expression (e.g., Contains, StartsWith). + /// + /// The method call expression. + /// The list to add the extracted clause to. + private static void ExtractMethodCallClause(MethodCallExpression methodCall, List filterClauses) + { + if (methodCall.Method.Name == "Contains") + { + // Check if this is property.Contains(value) or array.Contains(property) + if (methodCall.Object is MemberExpression member) + { + // This is property.Contains(value) - e.g., page.IncludeDomain.Contains("wikipedia.org") + var propertyName = member.Member.Name; + var value = ExtractValue(methodCall.Arguments[0]); + + if (value != null) + { + // For Contains, we'll map it to equality for domains (Tavily supports domain filtering) + if (propertyName.EndsWith("Domain", StringComparison.OrdinalIgnoreCase)) + { + filterClauses.Add(new EqualToFilterClause(propertyName, value)); + } + else if (propertyName.Equals("Title", StringComparison.OrdinalIgnoreCase)) + { + // For Title.Contains(), add the term to the search query itself + filterClauses.Add(new SearchQueryFilterClause(value.ToString() ?? string.Empty)); + } + else + { + throw new NotSupportedException($"Contains method is only supported for domain properties (IncludeDomain, ExcludeDomain) and Title, not '{propertyName}'."); + } + } + } + else if (methodCall.Object == null && methodCall.Arguments.Count == 2) + { + // This is array.Contains(property) - e.g., new[] { "general", "news" }.Contains(page.Topic) + // This pattern is not supported regardless of whether it's Enumerable.Contains (C# 13-) or MemoryExtensions.Contains (C# 14+) + // Both resolve to extension method calls with methodCall.Object == null + + // Provide detailed error message that covers both C# language versions + string errorMessage = "Collection Contains filters (e.g., array.Contains(page.Property)) are not supported by Tavily Search API. " + + "Tavily's API does not support OR logic across multiple values. "; + + if (IsMemoryExtensionsContains(methodCall)) + { + errorMessage += "Note: This occurs when using C# 14+ language features with span-based Contains methods (MemoryExtensions.Contains). "; + } + else + { + errorMessage += "Note: This occurs with standard LINQ extension methods (Enumerable.Contains). "; + } + + errorMessage += "Consider either: (1) performing multiple separate searches for each value, or " + + "(2) retrieving broader results and filtering on the client side."; + + throw new NotSupportedException(errorMessage); + } + else + { + throw new NotSupportedException("Unsupported Contains expression format."); + } + } + else + { + throw new NotSupportedException($"Method '{methodCall.Method.Name}' is not supported in Tavily search filters. Only 'Contains' is supported."); + } + } + + /// + /// Extracts a constant value from an expression. + /// + /// The expression to extract the value from. + /// The extracted value, or null if extraction failed. + private static object? ExtractValue(Expression expression) + { + return expression switch + { + ConstantExpression constant => constant.Value, + MemberExpression member when member.Expression is ConstantExpression constantExpr => + member.Member switch + { + System.Reflection.FieldInfo field => field.GetValue(constantExpr.Value), + System.Reflection.PropertyInfo property => property.GetValue(constantExpr.Value), + _ => null + }, + _ => Expression.Lambda(expression).Compile().DynamicInvoke() + }; + } + + #endregion + + #region Private Methods private readonly ILogger _logger; private readonly HttpClient _httpClient; @@ -175,6 +602,41 @@ private async IAsyncEnumerable GetSearchResultsAsync(TavilySearchRespons } } + /// + /// Return the search results as instances of . + /// + /// Response containing the web pages matching the query. + /// Cancellation token + private async IAsyncEnumerable GetResultsAsWebPageAsync(TavilySearchResponse? searchResponse, [EnumeratorCancellation] CancellationToken cancellationToken) + { + if (searchResponse is null || searchResponse.Results is null) + { + yield break; + } + + foreach (var result in searchResponse.Results) + { + yield return TavilyWebPage.FromSearchResult(result); + await Task.Yield(); + } + + if (this._searchOptions?.IncludeImages ?? false && searchResponse.Images is not null) + { + foreach (var image in searchResponse.Images!) + { + //For images, create a basic TavilyWebPage representation + Uri? imageUri = string.IsNullOrWhiteSpace(image.Url) ? null : new Uri(image.Url); + yield return new TavilyWebPage( + title: "Image Result", + url: imageUri, + content: image.Description ?? string.Empty, + score: 0.0 + ); + await Task.Yield(); + } + } + } + /// /// Return the search results as instances of . /// @@ -381,5 +843,40 @@ private TavilySearchRequest BuildRequestContent(string query, TextSearchOptions string strPayload = payload as string ?? JsonSerializer.Serialize(payload, s_jsonOptionsCache); return new(strPayload, Encoding.UTF8, "application/json"); } + + /// + /// Determines if a method call expression is a MemoryExtensions.Contains call (C# 14+ compatibility). + /// In C# 14+, array.Contains(property) may resolve to MemoryExtensions.Contains instead of Enumerable.Contains. + /// + /// The method call expression to check. + /// True if this is a MemoryExtensions.Contains call, false otherwise. + private static bool IsMemoryExtensionsContains(MethodCallExpression methodCall) + { + // Check if this is a static method call (Object is null) + if (methodCall.Object != null) + { + return false; + } + + // Check if it's MemoryExtensions.Contains + if (methodCall.Method.DeclaringType?.Name != "MemoryExtensions") + { + return false; + } + + // MemoryExtensions.Contains has 2-3 parameters: (ReadOnlySpan, T) or (ReadOnlySpan, T, IEqualityComparer) + if (methodCall.Arguments.Count < 2 || methodCall.Arguments.Count > 3) + { + return false; + } // For our text search scenarios, we don't support span comparers + if (methodCall.Arguments.Count == 3) + { + throw new NotSupportedException( + "MemoryExtensions.Contains with custom IEqualityComparer is not supported. " + + "Use simple array.Contains(property) expressions without custom comparers."); + } + + return true; + } #endregion } diff --git a/dotnet/src/Plugins/Plugins.Web/Tavily/TavilyWebPage.cs b/dotnet/src/Plugins/Plugins.Web/Tavily/TavilyWebPage.cs new file mode 100644 index 000000000000..fddf338e1e06 --- /dev/null +++ b/dotnet/src/Plugins/Plugins.Web/Tavily/TavilyWebPage.cs @@ -0,0 +1,102 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; + +namespace Microsoft.SemanticKernel.Plugins.Web.Tavily; + +/// +/// Represents a type-safe web page result from Tavily search for use with generic ITextSearch<TRecord> interface. +/// This class provides compile-time type safety and IntelliSense support for Tavily search filtering. +/// +public sealed class TavilyWebPage +{ + /// + /// Gets or sets the title of the web page. + /// + public string? Title { get; set; } + + /// + /// Gets or sets the URL of the web page. + /// + public Uri? Url { get; set; } + + /// + /// Gets or sets the content/description of the web page. + /// + public string? Content { get; set; } + + /// + /// Gets or sets the raw content of the web page (if available). + /// + public string? RawContent { get; set; } + + /// + /// Gets or sets the relevance score of the search result. + /// + public double Score { get; set; } + + /// + /// Gets or sets the topic filter for search results. + /// Maps to Tavily's 'topic' parameter for focused search. + /// + public string? Topic { get; set; } + + /// + /// Gets or sets the time range filter for search results. + /// Maps to Tavily's 'time_range' parameter (e.g., "day", "week", "month", "year"). + /// + public string? TimeRange { get; set; } + + /// + /// Gets or sets the number of days for time-based filtering. + /// Maps to Tavily's 'days' parameter for custom date ranges. + /// + public int? Days { get; set; } + + /// + /// Gets or sets the domain to include in search results. + /// Maps to Tavily's 'include_domain' parameter. + /// + public string? IncludeDomain { get; set; } + + /// + /// Gets or sets the domain to exclude from search results. + /// Maps to Tavily's 'exclude_domain' parameter. + /// + public string? ExcludeDomain { get; set; } + + /// + /// Initializes a new instance of the class. + /// + public TavilyWebPage() + { + } + + /// + /// Initializes a new instance of the class with specified values. + /// + /// The title of the web page. + /// The URL of the web page. + /// The content/description of the web page. + /// The relevance score. + /// The raw content (optional). + public TavilyWebPage(string? title, Uri? url, string? content, double score, string? rawContent = null) + { + this.Title = title; + this.Url = url; + this.Content = content; + this.Score = score; + this.RawContent = rawContent; + } + + /// + /// Creates a TavilyWebPage from a TavilySearchResult. + /// + /// The search result to convert. + /// A new TavilyWebPage instance. + internal static TavilyWebPage FromSearchResult(TavilySearchResult result) + { + Uri? url = string.IsNullOrWhiteSpace(result.Url) ? null : new Uri(result.Url); + return new TavilyWebPage(result.Title, url, result.Content, result.Score, result.RawContent); + } +} diff --git a/dotnet/src/Plugins/Plugins.Web/WebServiceCollectionExtensions.cs b/dotnet/src/Plugins/Plugins.Web/WebServiceCollectionExtensions.cs index e534ad5d2399..d4d004f70170 100644 --- a/dotnet/src/Plugins/Plugins.Web/WebServiceCollectionExtensions.cs +++ b/dotnet/src/Plugins/Plugins.Web/WebServiceCollectionExtensions.cs @@ -1,5 +1,7 @@ // Copyright (c) Microsoft. All rights reserved. +#pragma warning disable CS0618 // ITextSearch is obsolete - these extension methods provide backward compatibility + using Microsoft.Extensions.DependencyInjection; using Microsoft.SemanticKernel.Data; using Microsoft.SemanticKernel.Plugins.Web.Bing; diff --git a/dotnet/src/SemanticKernel.Abstractions/Data/TextSearch/ITextSearch.cs b/dotnet/src/SemanticKernel.Abstractions/Data/TextSearch/ITextSearch.cs index bb348a158c79..e955af86bc6c 100644 --- a/dotnet/src/SemanticKernel.Abstractions/Data/TextSearch/ITextSearch.cs +++ b/dotnet/src/SemanticKernel.Abstractions/Data/TextSearch/ITextSearch.cs @@ -1,13 +1,57 @@ // Copyright (c) Microsoft. All rights reserved. +using System.Diagnostics.CodeAnalysis; using System.Threading; using System.Threading.Tasks; namespace Microsoft.SemanticKernel.Data; +/// +/// Interface for text based search queries with type-safe LINQ filtering for use with Semantic Kernel prompts and automatic function calling. +/// +/// The type of record being searched. +[Experimental("SKEXP0001")] +public interface ITextSearch +{ + /// + /// Perform a search for content related to the specified query and return values representing the search results. + /// + /// What to search for. + /// Options used when executing a text search. + /// The to monitor for cancellation requests. The default is . + Task> SearchAsync( + string query, + TextSearchOptions? searchOptions = null, + CancellationToken cancellationToken = default); + + /// + /// Perform a search for content related to the specified query and return values representing the search results. + /// + /// What to search for. + /// Options used when executing a text search. + /// The to monitor for cancellation requests. The default is . + Task> GetTextSearchResultsAsync( + string query, + TextSearchOptions? searchOptions = null, + CancellationToken cancellationToken = default); + + /// + /// Perform a search for content related to the specified query and return strongly-typed values representing the search results. + /// + /// What to search for. + /// Options used when executing a text search. + /// The to monitor for cancellation requests. The default is . + Task> GetSearchResultsAsync( + string query, + TextSearchOptions? searchOptions = null, + CancellationToken cancellationToken = default); +} + /// /// Interface for text based search queries for use with Semantic Kernel prompts and automatic function calling. +/// This non-generic interface uses legacy for backward compatibility. /// +[System.Obsolete("Use ITextSearch with LINQ-based filtering instead. This interface will be removed in a future version.")] public interface ITextSearch { /// diff --git a/dotnet/src/SemanticKernel.Abstractions/Data/TextSearch/TextSearchOptions.cs b/dotnet/src/SemanticKernel.Abstractions/Data/TextSearch/TextSearchOptions.cs index cc995af02e8d..9375d34abd0f 100644 --- a/dotnet/src/SemanticKernel.Abstractions/Data/TextSearch/TextSearchOptions.cs +++ b/dotnet/src/SemanticKernel.Abstractions/Data/TextSearch/TextSearchOptions.cs @@ -1,6 +1,52 @@ // Copyright (c) Microsoft. All rights reserved. + +using System; +using System.Diagnostics.CodeAnalysis; +using System.Linq.Expressions; + namespace Microsoft.SemanticKernel.Data; +/// +/// Options which can be applied when using . +/// +/// The type of record being searched. +[Experimental("SKEXP0001")] +public sealed class TextSearchOptions +{ + /// + /// Default number of search results to return. + /// + public static readonly int DefaultTop = 5; + + /// + /// Flag indicating the total count should be included in the results. + /// + /// + /// Default value is false. + /// Not all text search implementations will support this option. + /// + public bool IncludeTotalCount { get; init; } = false; + + /// + /// The LINQ-based filter expression to apply to the search query. + /// + /// + /// This uses modern LINQ expressions for type-safe filtering, providing + /// compile-time safety and IntelliSense support. + /// + public Expression>? Filter { get; init; } + + /// + /// Number of search results to return. + /// + public int Top { get; init; } = DefaultTop; + + /// + /// The index of the first result to return. + /// + public int Skip { get; init; } = 0; +} + /// /// Options which can be applied when using . /// diff --git a/dotnet/src/SemanticKernel.AotTests/UnitTests/Search/MockTextSearch.cs b/dotnet/src/SemanticKernel.AotTests/UnitTests/Search/MockTextSearch.cs index 72aa218239f9..9ed0d43a87fa 100644 --- a/dotnet/src/SemanticKernel.AotTests/UnitTests/Search/MockTextSearch.cs +++ b/dotnet/src/SemanticKernel.AotTests/UnitTests/Search/MockTextSearch.cs @@ -4,7 +4,9 @@ namespace SemanticKernel.AotTests.UnitTests.Search; +#pragma warning disable CS0618 // Type or member is obsolete internal sealed class MockTextSearch : ITextSearch +#pragma warning restore CS0618 // Type or member is obsolete { private readonly KernelSearchResults? _objectResults; private readonly KernelSearchResults? _textSearchResults; diff --git a/dotnet/src/SemanticKernel.AotTests/UnitTests/Search/TextSearchExtensionsTests.cs b/dotnet/src/SemanticKernel.AotTests/UnitTests/Search/TextSearchExtensionsTests.cs index 8aff74675ecf..163b0294f5c1 100644 --- a/dotnet/src/SemanticKernel.AotTests/UnitTests/Search/TextSearchExtensionsTests.cs +++ b/dotnet/src/SemanticKernel.AotTests/UnitTests/Search/TextSearchExtensionsTests.cs @@ -21,7 +21,9 @@ public static async Task CreateWithSearch() // Arrange var testData = new List { "test-value" }; KernelSearchResults results = new(testData.ToAsyncEnumerable()); +#pragma warning disable CS0618 // Type or member is obsolete ITextSearch textSearch = new MockTextSearch(results); +#pragma warning restore CS0618 // Type or member is obsolete // Act var plugin = textSearch.CreateWithSearch("SearchPlugin", s_jsonSerializerOptions); @@ -35,7 +37,9 @@ public static async Task CreateWithGetTextSearchResults() // Arrange var testData = new List { new("test-value") }; KernelSearchResults results = new(testData.ToAsyncEnumerable()); +#pragma warning disable CS0618 // Type or member is obsolete ITextSearch textSearch = new MockTextSearch(results); +#pragma warning restore CS0618 // Type or member is obsolete // Act var plugin = textSearch.CreateWithGetTextSearchResults("SearchPlugin", s_jsonSerializerOptions); @@ -49,7 +53,9 @@ public static async Task CreateWithGetSearchResults() // Arrange var testData = new List { new("test-value") }; KernelSearchResults results = new(testData.ToAsyncEnumerable()); +#pragma warning disable CS0618 // Type or member is obsolete ITextSearch textSearch = new MockTextSearch(results); +#pragma warning restore CS0618 // Type or member is obsolete // Act var plugin = textSearch.CreateWithGetSearchResults("SearchPlugin", s_jsonSerializerOptions); diff --git a/dotnet/src/SemanticKernel.Core/Data/TextSearch/TextSearchExtensions.cs b/dotnet/src/SemanticKernel.Core/Data/TextSearch/TextSearchExtensions.cs index bfb829c44759..c326b939dca2 100644 --- a/dotnet/src/SemanticKernel.Core/Data/TextSearch/TextSearchExtensions.cs +++ b/dotnet/src/SemanticKernel.Core/Data/TextSearch/TextSearchExtensions.cs @@ -1,5 +1,7 @@ // Copyright (c) Microsoft. All rights reserved. +#pragma warning disable CS0618 // ITextSearch is obsolete - these extension methods provide backward compatibility + using System.Collections.Generic; using System.Diagnostics.CodeAnalysis; using System.Linq; diff --git a/dotnet/src/SemanticKernel.Core/Data/TextSearch/VectorStoreTextSearch.cs b/dotnet/src/SemanticKernel.Core/Data/TextSearch/VectorStoreTextSearch.cs index c92c86230efd..f1b18483c43a 100644 --- a/dotnet/src/SemanticKernel.Core/Data/TextSearch/VectorStoreTextSearch.cs +++ b/dotnet/src/SemanticKernel.Core/Data/TextSearch/VectorStoreTextSearch.cs @@ -16,7 +16,9 @@ namespace Microsoft.SemanticKernel.Data; /// A Vector Store Text Search implementation that can be used to perform searches using a . /// [Experimental("SKEXP0001")] -public sealed class VectorStoreTextSearch<[DynamicallyAccessedMembers(DynamicallyAccessedMemberTypes.PublicProperties)] TRecord> : ITextSearch +#pragma warning disable CS0618 // ITextSearch is obsolete - this class provides backward compatibility +public sealed class VectorStoreTextSearch<[DynamicallyAccessedMembers(DynamicallyAccessedMemberTypes.PublicProperties)] TRecord> : ITextSearch, ITextSearch +#pragma warning restore CS0618 #pragma warning restore CA1711 // Identifiers should not have incorrect suffix { /// @@ -194,6 +196,30 @@ public Task> GetSearchResultsAsync(string query, Tex return Task.FromResult(new KernelSearchResults(this.GetResultsAsRecordAsync(searchResponse, cancellationToken))); } + /// + Task> ITextSearch.SearchAsync(string query, TextSearchOptions? searchOptions, CancellationToken cancellationToken) + { + var searchResponse = this.ExecuteVectorSearchAsync(query, searchOptions, cancellationToken); + + return Task.FromResult(new KernelSearchResults(this.GetResultsAsStringAsync(searchResponse, cancellationToken))); + } + + /// + Task> ITextSearch.GetTextSearchResultsAsync(string query, TextSearchOptions? searchOptions, CancellationToken cancellationToken) + { + var searchResponse = this.ExecuteVectorSearchAsync(query, searchOptions, cancellationToken); + + return Task.FromResult(new KernelSearchResults(this.GetResultsAsTextSearchResultAsync(searchResponse, cancellationToken))); + } + + /// + Task> ITextSearch.GetSearchResultsAsync(string query, TextSearchOptions? searchOptions, CancellationToken cancellationToken) + { + var searchResponse = this.ExecuteVectorSearchAsync(query, searchOptions, cancellationToken); + + return Task.FromResult(new KernelSearchResults(this.GetResultsAsTRecordAsync(searchResponse, cancellationToken))); + } + #region private [Obsolete("This property is obsolete.")] private readonly ITextEmbeddingGenerationService? _textEmbeddingGeneration; @@ -244,28 +270,67 @@ private TextSearchStringMapper CreateTextSearchStringMapper() } /// - /// Execute a vector search and return the results. + /// Execute a vector search and return the results using legacy filtering for backward compatibility. /// /// What to search for. - /// Search options. + /// Search options with legacy TextSearchFilter. /// The to monitor for cancellation requests. The default is . private async IAsyncEnumerable> ExecuteVectorSearchAsync(string query, TextSearchOptions? searchOptions, [EnumeratorCancellation] CancellationToken cancellationToken) { searchOptions ??= new TextSearchOptions(); + var vectorSearchOptions = new VectorSearchOptions { #pragma warning disable CS0618 // VectorSearchFilter is obsolete - OldFilter = searchOptions.Filter?.FilterClauses is not null ? new VectorSearchFilter(searchOptions.Filter.FilterClauses) : null, -#pragma warning restore CS0618 // VectorSearchFilter is obsolete + OldFilter = searchOptions.Filter?.FilterClauses is not null + ? new VectorSearchFilter(searchOptions.Filter.FilterClauses) + : null, +#pragma warning restore CS0618 + Skip = searchOptions.Skip, + }; + + await foreach (var result in this.ExecuteVectorSearchCoreAsync(query, vectorSearchOptions, searchOptions.Top, cancellationToken).ConfigureAwait(false)) + { + yield return result; + } + } + + /// + /// Execute a vector search and return the results using modern LINQ filtering. + /// + /// What to search for. + /// Search options with LINQ filtering. + /// The to monitor for cancellation requests. The default is . + private async IAsyncEnumerable> ExecuteVectorSearchAsync(string query, TextSearchOptions? searchOptions, [EnumeratorCancellation] CancellationToken cancellationToken) + { + searchOptions ??= new TextSearchOptions(); + var vectorSearchOptions = new VectorSearchOptions + { + Filter = searchOptions.Filter, // Use modern LINQ filtering directly Skip = searchOptions.Skip, }; + await foreach (var result in this.ExecuteVectorSearchCoreAsync(query, vectorSearchOptions, searchOptions.Top, cancellationToken).ConfigureAwait(false)) + { + yield return result; + } + } + + /// + /// Core vector search execution logic. + /// + /// What to search for. + /// Vector search options. + /// Maximum number of results to return. + /// The to monitor for cancellation requests. + private async IAsyncEnumerable> ExecuteVectorSearchCoreAsync(string query, VectorSearchOptions vectorSearchOptions, int top, [EnumeratorCancellation] CancellationToken cancellationToken) + { #pragma warning disable CS0618 // Type or member is obsolete if (this._textEmbeddingGeneration is not null) { var vectorizedQuery = await this._textEmbeddingGeneration!.GenerateEmbeddingAsync(query, cancellationToken: cancellationToken).ConfigureAwait(false); - await foreach (var result in this._vectorSearchable!.SearchAsync(vectorizedQuery, searchOptions.Top, vectorSearchOptions, cancellationToken).ConfigureAwait(false)) + await foreach (var result in this._vectorSearchable!.SearchAsync(vectorizedQuery, top, vectorSearchOptions, cancellationToken).WithCancellation(cancellationToken).ConfigureAwait(false)) { yield return result; } @@ -274,7 +339,7 @@ private async IAsyncEnumerable> ExecuteVectorSearchA } #pragma warning restore CS0618 // Type or member is obsolete - await foreach (var result in this._vectorSearchable!.SearchAsync(query, searchOptions.Top, vectorSearchOptions, cancellationToken).ConfigureAwait(false)) + await foreach (var result in this._vectorSearchable!.SearchAsync(query, top, vectorSearchOptions, cancellationToken).WithCancellation(cancellationToken).ConfigureAwait(false)) { yield return result; } @@ -302,6 +367,28 @@ private async IAsyncEnumerable GetResultsAsRecordAsync(IAsyncEnumerable< } } + /// + /// Return the search results as strongly-typed instances. + /// + /// Response containing the records matching the query. + /// Cancellation token + private async IAsyncEnumerable GetResultsAsTRecordAsync(IAsyncEnumerable>? searchResponse, [EnumeratorCancellation] CancellationToken cancellationToken) + { + if (searchResponse is null) + { + yield break; + } + + await foreach (var result in searchResponse.WithCancellation(cancellationToken).ConfigureAwait(false)) + { + if (result.Record is not null) + { + yield return result.Record; + await Task.Yield(); + } + } + } + /// /// Return the search results as instances of . /// diff --git a/dotnet/src/SemanticKernel.Core/Data/TextSearchBehavior/TextSearchProvider.cs b/dotnet/src/SemanticKernel.Core/Data/TextSearchBehavior/TextSearchProvider.cs index 6ee680d91826..fe6a9f7d0d35 100644 --- a/dotnet/src/SemanticKernel.Core/Data/TextSearchBehavior/TextSearchProvider.cs +++ b/dotnet/src/SemanticKernel.Core/Data/TextSearchBehavior/TextSearchProvider.cs @@ -1,5 +1,7 @@ // Copyright (c) Microsoft. All rights reserved. +#pragma warning disable CS0618 // ITextSearch is obsolete - this class provides backward compatibility + using System; using System.Collections.Generic; using System.Diagnostics.CodeAnalysis; diff --git a/dotnet/src/SemanticKernel.Core/Data/TextSearchStore/TextSearchStore.cs b/dotnet/src/SemanticKernel.Core/Data/TextSearchStore/TextSearchStore.cs index ed2314eb8b1e..d1d22aacab34 100644 --- a/dotnet/src/SemanticKernel.Core/Data/TextSearchStore/TextSearchStore.cs +++ b/dotnet/src/SemanticKernel.Core/Data/TextSearchStore/TextSearchStore.cs @@ -1,5 +1,7 @@ // Copyright (c) Microsoft. All rights reserved. +#pragma warning disable CS0618 // ITextSearch is obsolete - this class provides backward compatibility + using System; using System.Collections.Generic; using System.Diagnostics.CodeAnalysis; diff --git a/dotnet/src/SemanticKernel.UnitTests/Data/MockTextSearch.cs b/dotnet/src/SemanticKernel.UnitTests/Data/MockTextSearch.cs index 916b158fc770..01746adf623e 100644 --- a/dotnet/src/SemanticKernel.UnitTests/Data/MockTextSearch.cs +++ b/dotnet/src/SemanticKernel.UnitTests/Data/MockTextSearch.cs @@ -10,7 +10,9 @@ namespace SemanticKernel.UnitTests.Data; /// /// Mock implementation of /// +#pragma warning disable CS0618 // Type or member is obsolete internal sealed class MockTextSearch(int count = 3, long totalCount = 30) : ITextSearch +#pragma warning restore CS0618 // Type or member is obsolete { /// public Task> GetSearchResultsAsync(string query, TextSearchOptions? searchOptions = null, CancellationToken cancellationToken = default) diff --git a/dotnet/src/SemanticKernel.UnitTests/Data/TextSearchProviderTests.cs b/dotnet/src/SemanticKernel.UnitTests/Data/TextSearchProviderTests.cs index 28d37124a3c9..c552a426d272 100644 --- a/dotnet/src/SemanticKernel.UnitTests/Data/TextSearchProviderTests.cs +++ b/dotnet/src/SemanticKernel.UnitTests/Data/TextSearchProviderTests.cs @@ -1,5 +1,7 @@ // Copyright (c) Microsoft. All rights reserved. +#pragma warning disable CS0618 // Type or member is obsolete - Testing legacy non-generic ITextSearch interface + using System; using System.Collections.Generic; using System.Linq; diff --git a/dotnet/src/SemanticKernel.UnitTests/Data/VectorStoreTextSearchTestBase.cs b/dotnet/src/SemanticKernel.UnitTests/Data/VectorStoreTextSearchTestBase.cs index ec0134936f3f..066cf7ef2398 100644 --- a/dotnet/src/SemanticKernel.UnitTests/Data/VectorStoreTextSearchTestBase.cs +++ b/dotnet/src/SemanticKernel.UnitTests/Data/VectorStoreTextSearchTestBase.cs @@ -140,6 +140,7 @@ public string MapFromResultToString(object result) { DataModel dataModel => dataModel.Text, DataModelWithRawEmbedding dataModelWithRawEmbedding => dataModelWithRawEmbedding.Text, + DataModelWithTags dataModelWithTags => dataModelWithTags.Text, _ => throw new ArgumentException("Invalid result type.") }; } @@ -155,6 +156,7 @@ public TextSearchResult MapFromResultToTextSearchResult(object result) { DataModel dataModel => new TextSearchResult(value: dataModel.Text) { Name = dataModel.Key.ToString() }, DataModelWithRawEmbedding dataModelWithRawEmbedding => new TextSearchResult(value: dataModelWithRawEmbedding.Text) { Name = dataModelWithRawEmbedding.Key.ToString() }, + DataModelWithTags dataModelWithTags => new TextSearchResult(value: dataModelWithTags.Text) { Name = dataModelWithTags.Key.ToString() }, _ => throw new ArgumentException("Invalid result type.") }; } @@ -231,4 +233,27 @@ public sealed class DataModelWithRawEmbedding [VectorStoreVector(1536)] public ReadOnlyMemory Embedding { get; init; } } + + /// + /// Sample model class for testing collection-based filtering (AnyTagEqualTo). + /// +#pragma warning disable CA1812 // Avoid uninstantiated internal classes + public sealed class DataModelWithTags +#pragma warning restore CA1812 // Avoid uninstantiated internal classes + { + [VectorStoreKey] + public Guid Key { get; init; } + + [VectorStoreData] + public required string Text { get; init; } + + [VectorStoreData(IsIndexed = true)] + public required string Tag { get; init; } + + [VectorStoreData(IsIndexed = true)] + public required IReadOnlyList Tags { get; init; } + + [VectorStoreVector(1536)] + public string? Embedding { get; init; } + } } diff --git a/dotnet/src/SemanticKernel.UnitTests/Data/VectorStoreTextSearchTests.cs b/dotnet/src/SemanticKernel.UnitTests/Data/VectorStoreTextSearchTests.cs index 66803cc86f53..75f4b090590e 100644 --- a/dotnet/src/SemanticKernel.UnitTests/Data/VectorStoreTextSearchTests.cs +++ b/dotnet/src/SemanticKernel.UnitTests/Data/VectorStoreTextSearchTests.cs @@ -9,6 +9,7 @@ using Xunit; namespace SemanticKernel.UnitTests.Data; + public class VectorStoreTextSearchTests : VectorStoreTextSearchTestBase { #pragma warning disable CS0618 // VectorStoreTextSearch with ITextEmbeddingGenerationService is obsolete @@ -77,12 +78,14 @@ public async Task CanGetSearchResultAsync() { // Arrange. var sut = await CreateVectorStoreTextSearchAsync(); + ITextSearch typeSafeInterface = sut; // Act. - KernelSearchResults searchResults = await sut.GetSearchResultsAsync("What is the Semantic Kernel?", new() { Top = 2, Skip = 0 }); + KernelSearchResults searchResults = await typeSafeInterface.GetSearchResultsAsync("What is the Semantic Kernel?", new TextSearchOptions { Top = 2, Skip = 0 }); var results = await searchResults.Results.ToListAsync(); Assert.Equal(2, results.Count); + Assert.All(results, result => Assert.IsType(result)); } [Fact] @@ -116,12 +119,14 @@ public async Task CanGetSearchResultsWithEmbeddingGeneratorAsync() { // Arrange. var sut = await CreateVectorStoreTextSearchWithEmbeddingGeneratorAsync(); + ITextSearch typeSafeInterface = sut; // Act. - KernelSearchResults searchResults = await sut.GetSearchResultsAsync("What is the Semantic Kernel?", new() { Top = 2, Skip = 0 }); + KernelSearchResults searchResults = await typeSafeInterface.GetSearchResultsAsync("What is the Semantic Kernel?", new TextSearchOptions { Top = 2, Skip = 0 }); var results = await searchResults.Results.ToListAsync(); Assert.Equal(2, results.Count); + Assert.All(results, result => Assert.IsType(result)); } #pragma warning disable CS0618 // VectorStoreTextSearch with ITextEmbeddingGenerationService is obsolete @@ -203,4 +208,221 @@ public async Task CanFilterGetSearchResultsWithVectorizedSearchAsync() result2 = oddResults[1] as DataModel; Assert.Equal("Odd", result2?.Tag); } + + #region Generic Interface Tests (ITextSearch) + + [Fact] + public async Task LinqSearchAsync() + { + // Arrange - Create VectorStoreTextSearch (implements both interfaces) + var sut = await CreateVectorStoreTextSearchAsync(); + + // Cast to ITextSearch to use type-safe LINQ filtering + ITextSearch typeSafeInterface = sut; + + // Act - Use generic interface with LINQ filter + var searchOptions = new TextSearchOptions + { + Top = 5, + Filter = r => r.Tag == "Even" + }; + + KernelSearchResults searchResults = await typeSafeInterface.SearchAsync( + "What is the Semantic Kernel?", + searchOptions); + var results = await searchResults.Results.ToListAsync(); + + // Assert - Should return results (filtering applied at vector store level) + Assert.NotEmpty(results); + } + + [Fact] + public async Task LinqGetTextSearchResultsAsync() + { + // Arrange + var sut = await CreateVectorStoreTextSearchAsync(); + ITextSearch typeSafeInterface = sut; + + // Act - Use generic interface with LINQ filter + var searchOptions = new TextSearchOptions + { + Top = 5, + Filter = r => r.Tag == "Odd" + }; + + KernelSearchResults searchResults = await typeSafeInterface.GetTextSearchResultsAsync( + "What is the Semantic Kernel?", + searchOptions); + var results = await searchResults.Results.ToListAsync(); + + // Assert + Assert.NotEmpty(results); + Assert.All(results, result => Assert.NotNull(result.Value)); + } + + [Fact] + public async Task LinqGetSearchResultsAsync() + { + // Arrange + var sut = await CreateVectorStoreTextSearchAsync(); + ITextSearch typeSafeInterface = sut; + + // Act - Use type-safe LINQ filtering with ITextSearch + var searchOptions = new TextSearchOptions + { + Top = 5, + Filter = r => r.Tag == "Even" + }; + + KernelSearchResults searchResults = await typeSafeInterface.GetSearchResultsAsync( + "What is the Semantic Kernel?", + searchOptions); + var results = await searchResults.Results.ToListAsync(); + + // Assert - Results should be strongly-typed DataModel objects with Tag == "Even" + Assert.NotEmpty(results); + Assert.All(results, result => + { + Assert.Equal("Even", result.Tag); // Direct property access - no cast needed! + }); + } + + [Fact] + public async Task LinqFilterSimpleEqualityAsync() + { + // Arrange + var sut = await CreateVectorStoreTextSearchAsync(); + ITextSearch typeSafeInterface = sut; + + // Act - Simple equality filter + var searchOptions = new TextSearchOptions + { + Top = 10, + Filter = r => r.Tag == "Odd" + }; + + var searchResults = await typeSafeInterface.GetSearchResultsAsync("test", searchOptions); + var results = await searchResults.Results.ToListAsync(); + + // Assert - All results should have Tag == "Odd" + Assert.NotEmpty(results); + Assert.All(results.Cast(), dm => Assert.Equal("Odd", dm.Tag)); + } + + [Fact] + public async Task LinqFilterComplexExpressionAsync() + { + // Arrange + var sut = await CreateVectorStoreTextSearchAsync(); + ITextSearch typeSafeInterface = sut; + + // Act - Complex LINQ expression with multiple conditions + var searchOptions = new TextSearchOptions + { + Top = 10, + Filter = r => r.Tag == "Even" && r.Text.Contains("Record") + }; + + var searchResults = await typeSafeInterface.GetSearchResultsAsync("test", searchOptions); + var results = await searchResults.Results.ToListAsync(); + + // Assert - Results should match both conditions + Assert.NotEmpty(results); + Assert.All(results.Cast(), dm => + { + Assert.Equal("Even", dm.Tag); + Assert.Contains("Record", dm.Text); + }); + } + + [Fact] + public async Task LinqFilterCollectionContainsAsync() + { + // Arrange - Create collection with DataModelWithTags + using var embeddingGenerator = new MockTextEmbeddingGenerator(); + using var vectorStore = new InMemoryVectorStore(new() { EmbeddingGenerator = embeddingGenerator }); + var collection = vectorStore.GetCollection("records"); + await collection.EnsureCollectionExistsAsync(); + + // Add test records with tags + var records = new[] + { + new DataModelWithTags + { + Key = Guid.NewGuid(), + Text = "First", + Tag = "test", + Tags = new[] { "important", "urgent" }, + Embedding = "First" + }, + new DataModelWithTags + { + Key = Guid.NewGuid(), + Text = "Second", + Tag = "test", + Tags = new[] { "normal", "routine" }, + Embedding = "Second" + }, + new DataModelWithTags + { + Key = Guid.NewGuid(), + Text = "Third", + Tag = "test", + Tags = new[] { "important", "routine" }, + Embedding = "Third" + } + }; + + foreach (var record in records) + { + await collection.UpsertAsync(record); + } + + var textSearch = new VectorStoreTextSearch( + collection, + new DataModelTextSearchStringMapper(), + new DataModelTextSearchResultMapper()); + + ITextSearch typeSafeInterface = textSearch; + + // Act - Use LINQ .Contains() for collection filtering + var searchOptions = new TextSearchOptions + { + Top = 10, + Filter = r => r.Tags.Contains("important") + }; + + var searchResults = await typeSafeInterface.GetSearchResultsAsync("test", searchOptions); + var results = await searchResults.Results.ToListAsync(); + + // Assert - Should return 2 records with "important" tag + Assert.Equal(2, results.Count); + Assert.All(results.Cast(), dm => + Assert.Contains("important", dm.Tags)); + } + + [Fact] + public async Task LinqFilterNullReturnsAllResultsAsync() + { + // Arrange + var sut = await CreateVectorStoreTextSearchAsync(); + ITextSearch typeSafeInterface = sut; + + // Act - Use generic interface with null filter + var searchOptions = new TextSearchOptions + { + Top = 10, + Filter = null // No filter + }; + + var searchResults = await typeSafeInterface.GetSearchResultsAsync("test", searchOptions); + var results = await searchResults.Results.ToListAsync(); + + // Assert - Should return both "Even" and "Odd" records + var dataModels = results.Cast().ToList(); + Assert.Contains(dataModels, dm => dm.Tag == "Even"); + Assert.Contains(dataModels, dm => dm.Tag == "Odd"); + } + + #endregion } diff --git a/dotnet/src/VectorData/VectorData.Abstractions/FilterClauses/FilterClause.cs b/dotnet/src/VectorData/VectorData.Abstractions/FilterClauses/FilterClause.cs index af0c1dac51b3..be72560ffc2f 100644 --- a/dotnet/src/VectorData/VectorData.Abstractions/FilterClauses/FilterClause.cs +++ b/dotnet/src/VectorData/VectorData.Abstractions/FilterClauses/FilterClause.cs @@ -11,7 +11,10 @@ namespace Microsoft.Extensions.VectorData; /// public abstract class FilterClause { - internal FilterClause() + /// + /// Initializes a new instance of the class. + /// + protected FilterClause() { } }