diff --git a/dotnet/src/Plugins/Plugins.UnitTests/Web/Bing/BingTextSearchTests.cs b/dotnet/src/Plugins/Plugins.UnitTests/Web/Bing/BingTextSearchTests.cs index a6172e334314..633c9ec5a888 100644 --- a/dotnet/src/Plugins/Plugins.UnitTests/Web/Bing/BingTextSearchTests.cs +++ b/dotnet/src/Plugins/Plugins.UnitTests/Web/Bing/BingTextSearchTests.cs @@ -231,6 +231,261 @@ public async Task DoesNotBuildsUriForInvalidQueryParameterAsync() Assert.Equal("Unknown equality filter clause field name 'fooBar', must be one of answerCount,cc,freshness,mkt,promote,responseFilter,safeSearch,setLang,textDecorations,textFormat,contains,ext,filetype,inanchor,inbody,intitle,ip,language,loc,location,prefer,site,feed,hasfeed,url (Parameter 'searchOptions')", e.Message); } + #region Generic ITextSearch Interface Tests + + [Fact] + public async Task GenericSearchAsyncWithLanguageEqualityFilterProducesCorrectBingQueryAsync() + { + // Arrange + this._messageHandlerStub.AddJsonResponse(File.ReadAllText(WhatIsTheSKResponseJson)); + ITextSearch textSearch = new BingTextSearch(apiKey: "ApiKey", options: new() { HttpClient = this._httpClient }); + + // Act + var searchOptions = new TextSearchOptions + { + Top = 4, + Skip = 0, + Filter = page => page.Language == "en" + }; + KernelSearchResults result = await textSearch.SearchAsync("What is the Semantic Kernel?", searchOptions); + + // Assert - Verify LINQ expression converted to Bing's language: operator + var requestUris = this._messageHandlerStub.RequestUris; + Assert.Single(requestUris); + Assert.NotNull(requestUris[0]); + Assert.Contains("language%3Aen", requestUris[0]!.AbsoluteUri); + Assert.Contains("count=4", requestUris[0]!.AbsoluteUri); + Assert.Contains("offset=0", requestUris[0]!.AbsoluteUri); + } + + [Fact] + public async Task GenericSearchAsyncWithLanguageInequalityFilterProducesCorrectBingQueryAsync() + { + // Arrange + this._messageHandlerStub.AddJsonResponse(File.ReadAllText(WhatIsTheSKResponseJson)); + ITextSearch textSearch = new BingTextSearch(apiKey: "ApiKey", options: new() { HttpClient = this._httpClient }); + + // Act + var searchOptions = new TextSearchOptions + { + Top = 4, + Skip = 0, + Filter = page => page.Language != "fr" + }; + KernelSearchResults result = await textSearch.SearchAsync("What is the Semantic Kernel?", searchOptions); + + // Assert - Verify LINQ inequality expression converted to Bing's negation syntax (-language:fr) + var requestUris = this._messageHandlerStub.RequestUris; + Assert.Single(requestUris); + Assert.NotNull(requestUris[0]); + Assert.Contains("-language%3Afr", requestUris[0]!.AbsoluteUri); + } + + [Fact] + public async Task GenericSearchAsyncWithContainsFilterProducesCorrectBingQueryAsync() + { + // Arrange + this._messageHandlerStub.AddJsonResponse(File.ReadAllText(WhatIsTheSKResponseJson)); + ITextSearch textSearch = new BingTextSearch(apiKey: "ApiKey", options: new() { HttpClient = this._httpClient }); + + // Act + var searchOptions = new TextSearchOptions + { + Top = 4, + Skip = 0, + Filter = page => page.Name.Contains("Microsoft") + }; + KernelSearchResults result = await textSearch.SearchAsync("What is the Semantic Kernel?", searchOptions); + + // Assert - Verify LINQ Contains() converted to Bing's intitle: operator + var requestUris = this._messageHandlerStub.RequestUris; + Assert.Single(requestUris); + Assert.NotNull(requestUris[0]); + Assert.Contains("intitle%3AMicrosoft", requestUris[0]!.AbsoluteUri); + } + + [Fact] + public async Task GenericSearchAsyncWithComplexAndFilterProducesCorrectBingQueryAsync() + { + // Arrange + this._messageHandlerStub.AddJsonResponse(File.ReadAllText(WhatIsTheSKResponseJson)); + ITextSearch textSearch = new BingTextSearch(apiKey: "ApiKey", options: new() { HttpClient = this._httpClient }); + + // Act + var searchOptions = new TextSearchOptions + { + Top = 4, + Skip = 0, + Filter = page => page.Language == "en" && page.Name.Contains("AI") + }; + KernelSearchResults result = await textSearch.SearchAsync("What is the Semantic Kernel?", searchOptions); + + // Assert - Verify LINQ AND expression produces both Bing operators + var requestUris = this._messageHandlerStub.RequestUris; + Assert.Single(requestUris); + Assert.NotNull(requestUris[0]); + Assert.Contains("language%3Aen", requestUris[0]!.AbsoluteUri); + Assert.Contains("intitle%3AAI", requestUris[0]!.AbsoluteUri); + } + + [Fact] + public async Task GenericGetTextSearchResultsAsyncWithUrlFilterProducesCorrectBingQueryAsync() + { + // Arrange + this._messageHandlerStub.AddJsonResponse(File.ReadAllText(WhatIsTheSKResponseJson)); + ITextSearch textSearch = new BingTextSearch(apiKey: "ApiKey", options: new() { HttpClient = this._httpClient }); + + // Act + var searchOptions = new TextSearchOptions + { + Top = 4, + Skip = 0, + Filter = page => page.Url.Contains("microsoft.com") + }; + KernelSearchResults result = await textSearch.GetTextSearchResultsAsync("What is the Semantic Kernel?", searchOptions); + + // Assert - Verify LINQ Url.Contains() converted to Bing's url: operator + var requestUris = this._messageHandlerStub.RequestUris; + Assert.Single(requestUris); + Assert.NotNull(requestUris[0]); + Assert.Contains("url%3Amicrosoft.com", requestUris[0]!.AbsoluteUri); + + // Also verify result structure + Assert.NotNull(result); + Assert.NotNull(result.Results); + } + + [Fact] + public async Task GenericGetSearchResultsAsyncWithSnippetContainsFilterProducesCorrectBingQueryAsync() + { + // Arrange + this._messageHandlerStub.AddJsonResponse(File.ReadAllText(WhatIsTheSKResponseJson)); + ITextSearch textSearch = new BingTextSearch(apiKey: "ApiKey", options: new() { HttpClient = this._httpClient }); + + // Act + var searchOptions = new TextSearchOptions + { + Top = 4, + Skip = 0, + Filter = page => page.Snippet.Contains("semantic") + }; + KernelSearchResults result = await textSearch.GetSearchResultsAsync("What is the Semantic Kernel?", searchOptions); + + // Assert - Verify LINQ Snippet.Contains() converted to Bing's inbody: operator + var requestUris = this._messageHandlerStub.RequestUris; + Assert.Single(requestUris); + Assert.NotNull(requestUris[0]); + Assert.Contains("inbody%3Asemantic", requestUris[0]!.AbsoluteUri); + + // Verify result structure + Assert.NotNull(result); + Assert.NotNull(result.Results); + } + + [Fact] + public async Task GenericSearchAsyncWithDisplayUrlEqualityFilterProducesCorrectBingQueryAsync() + { + // Arrange + this._messageHandlerStub.AddJsonResponse(File.ReadAllText(SiteFilterDevBlogsResponseJson)); + ITextSearch textSearch = new BingTextSearch(apiKey: "ApiKey", options: new() { HttpClient = this._httpClient }); + + // Act + var searchOptions = new TextSearchOptions + { + Top = 4, + Skip = 0, + Filter = page => page.DisplayUrl == "devblogs.microsoft.com" + }; + KernelSearchResults result = await textSearch.SearchAsync("What is the Semantic Kernel?", searchOptions); + + // Assert - Verify LINQ DisplayUrl equality converted to Bing's site: operator + var requestUris = this._messageHandlerStub.RequestUris; + Assert.Single(requestUris); + Assert.NotNull(requestUris[0]); + Assert.Contains("site%3Adevblogs.microsoft.com", requestUris[0]!.AbsoluteUri); + } + + [Fact] + public async Task GenericSearchAsyncWithMultipleAndConditionsProducesCorrectBingQueryAsync() + { + // Arrange + this._messageHandlerStub.AddJsonResponse(File.ReadAllText(WhatIsTheSKResponseJson)); + ITextSearch textSearch = new BingTextSearch(apiKey: "ApiKey", options: new() { HttpClient = this._httpClient }); + + // Act + var searchOptions = new TextSearchOptions + { + Top = 4, + Skip = 0, + Filter = page => page.Language == "en" && page.DisplayUrl.Contains("microsoft.com") && page.Name.Contains("Semantic") + }; + KernelSearchResults result = await textSearch.SearchAsync("What is the Semantic Kernel?", searchOptions); + + // Assert - Verify all LINQ conditions converted correctly + var requestUris = this._messageHandlerStub.RequestUris; + Assert.Single(requestUris); + Assert.NotNull(requestUris[0]); + string uri = requestUris[0]!.AbsoluteUri; + Assert.Contains("language%3Aen", uri); + Assert.Contains("site%3Amicrosoft.com", uri); // DisplayUrl.Contains() → site: operator + Assert.Contains("intitle%3ASemantic", uri); + } + + [Fact] + public async Task GenericSearchAsyncWithNoFilterReturnsResultsSuccessfullyAsync() + { + // Arrange + this._messageHandlerStub.AddJsonResponse(File.ReadAllText(WhatIsTheSKResponseJson)); + ITextSearch textSearch = new BingTextSearch(apiKey: "ApiKey", options: new() { HttpClient = this._httpClient }); + + // Act - No filter specified + var searchOptions = new TextSearchOptions + { + Top = 10, + Skip = 0 + }; + KernelSearchResults result = await textSearch.SearchAsync("What is the Semantic Kernel?", searchOptions); + + // Assert - Verify basic query without filter operators + var requestUris = this._messageHandlerStub.RequestUris; + Assert.Single(requestUris); + Assert.NotNull(requestUris[0]); + Assert.DoesNotContain("language%3A", requestUris[0]!.AbsoluteUri); + Assert.DoesNotContain("intitle%3A", requestUris[0]!.AbsoluteUri); + + // Verify results + Assert.NotNull(result); + Assert.NotNull(result.Results); + var resultList = await result.Results.ToListAsync(); + Assert.Equal(10, resultList.Count); + } + + [Fact] + public async Task GenericSearchAsyncWithIsFamilyFriendlyFilterProducesCorrectBingQueryAsync() + { + // Arrange + this._messageHandlerStub.AddJsonResponse(File.ReadAllText(WhatIsTheSKResponseJson)); + ITextSearch textSearch = new BingTextSearch(apiKey: "ApiKey", options: new() { HttpClient = this._httpClient }); + + // Act + var searchOptions = new TextSearchOptions + { + Top = 4, + Skip = 0, + Filter = page => page.IsFamilyFriendly == true + }; + KernelSearchResults result = await textSearch.SearchAsync("What is the Semantic Kernel?", searchOptions); + + // Assert - Verify LINQ IsFamilyFriendly equality converted to Bing's safeSearch query parameter + var requestUris = this._messageHandlerStub.RequestUris; + Assert.Single(requestUris); + Assert.NotNull(requestUris[0]); + // safeSearch is a query parameter, not an advanced search operator + Assert.Contains("safeSearch=true", requestUris[0]!.AbsoluteUri); + } + + #endregion + /// public void Dispose() { diff --git a/dotnet/src/Plugins/Plugins.Web/Bing/BingTextSearch.cs b/dotnet/src/Plugins/Plugins.Web/Bing/BingTextSearch.cs index 556e04f148d3..6cc5c8dee6f1 100644 --- a/dotnet/src/Plugins/Plugins.Web/Bing/BingTextSearch.cs +++ b/dotnet/src/Plugins/Plugins.Web/Bing/BingTextSearch.cs @@ -3,6 +3,7 @@ using System; using System.Collections.Generic; using System.Linq; +using System.Linq.Expressions; using System.Net.Http; using System.Runtime.CompilerServices; using System.Text; @@ -20,7 +21,7 @@ namespace Microsoft.SemanticKernel.Plugins.Web.Bing; /// /// A Bing Text Search implementation that can be used to perform searches using the Bing Web Search API. /// -public sealed class BingTextSearch : ITextSearch +public sealed class BingTextSearch : ITextSearch, ITextSearch { /// /// Create an instance of the with API key authentication. @@ -74,6 +75,27 @@ public async Task> GetSearchResultsAsync(string quer return new KernelSearchResults(this.GetResultsAsWebPageAsync(searchResponse, cancellationToken), totalCount, GetResultsMetadata(searchResponse)); } + /// + Task> ITextSearch.SearchAsync(string query, TextSearchOptions? searchOptions, CancellationToken cancellationToken) + { + var legacyOptions = searchOptions != null ? ConvertToLegacyOptions(searchOptions) : new TextSearchOptions(); + return this.SearchAsync(query, legacyOptions, cancellationToken); + } + + /// + Task> ITextSearch.GetTextSearchResultsAsync(string query, TextSearchOptions? searchOptions, CancellationToken cancellationToken) + { + var legacyOptions = searchOptions != null ? ConvertToLegacyOptions(searchOptions) : new TextSearchOptions(); + return this.GetTextSearchResultsAsync(query, legacyOptions, cancellationToken); + } + + /// + Task> ITextSearch.GetSearchResultsAsync(string query, TextSearchOptions? searchOptions, CancellationToken cancellationToken) + { + var legacyOptions = searchOptions != null ? ConvertToLegacyOptions(searchOptions) : new TextSearchOptions(); + return this.GetSearchResultsAsync(query, legacyOptions, cancellationToken); + } + #region private private readonly ILogger _logger; @@ -92,6 +114,249 @@ public async Task> GetSearchResultsAsync(string quer private const string DefaultUri = "https://api.bing.microsoft.com/v7.0/search"; + /// + /// Converts generic TextSearchOptions with LINQ filtering to legacy TextSearchOptions. + /// Attempts to translate simple LINQ expressions to Bing API filters where possible. + /// + /// The generic search options with LINQ filtering. + /// Legacy TextSearchOptions with equivalent filtering, or null if no conversion possible. + private static TextSearchOptions ConvertToLegacyOptions(TextSearchOptions genericOptions) + { + return new TextSearchOptions + { + Top = genericOptions.Top, + Skip = genericOptions.Skip, + Filter = genericOptions.Filter != null ? ConvertLinqExpressionToBingFilter(genericOptions.Filter) : null + }; + } + + /// + /// Converts a LINQ expression to a TextSearchFilter compatible with Bing API. + /// Supports equality, inequality, Contains() method calls, and logical AND operator. + /// + /// The LINQ expression to convert. + /// A TextSearchFilter with equivalent filtering. + /// Thrown when the expression cannot be converted to Bing filters. + private static TextSearchFilter ConvertLinqExpressionToBingFilter(Expression> linqExpression) + { + var filter = new TextSearchFilter(); + ProcessExpression(linqExpression.Body, filter); + return filter; + } + + /// + /// Recursively processes LINQ expression nodes and builds Bing API filters. + /// + private static void ProcessExpression(Expression expression, TextSearchFilter filter) + { + switch (expression) + { + case BinaryExpression binaryExpr when binaryExpr.NodeType == ExpressionType.AndAlso: + // Handle AND: page => page.Language == "en" && page.Name.Contains("AI") + ProcessExpression(binaryExpr.Left, filter); + ProcessExpression(binaryExpr.Right, filter); + break; + + case BinaryExpression binaryExpr when binaryExpr.NodeType == ExpressionType.OrElse: + // Handle OR: Currently not directly supported by TextSearchFilter + // Bing API supports OR via multiple queries, but TextSearchFilter doesn't expose this + throw new NotSupportedException( + "Logical OR (||) is not supported by Bing Text Search filters. " + + "Consider splitting into multiple search queries."); + + case UnaryExpression unaryExpr when unaryExpr.NodeType == ExpressionType.Not: + // Handle NOT: page => !page.Language.Equals("en") + throw new NotSupportedException( + "Logical NOT (!) is not directly supported by Bing Text Search advanced operators. " + + "Consider restructuring your filter to use positive conditions."); + + case BinaryExpression binaryExpr when binaryExpr.NodeType == ExpressionType.Equal: + // Handle equality: page => page.Language == "en" + ProcessEqualityExpression(binaryExpr, filter, isNegated: false); + break; + + case BinaryExpression binaryExpr when binaryExpr.NodeType == ExpressionType.NotEqual: + // Handle inequality: page => page.Language != "en" + // Implemented via Bing's negation syntax (e.g., -language:en) + ProcessEqualityExpression(binaryExpr, filter, isNegated: true); + break; + + case MethodCallExpression methodExpr when methodExpr.Method.Name == "Contains": + // Handle Contains: page => page.Name.Contains("Microsoft") + ProcessContainsExpression(methodExpr, filter); + break; + + default: + throw new NotSupportedException( + $"Expression type '{expression.NodeType}' is not supported for Bing API filters. " + + "Supported patterns: equality (==), inequality (!=), Contains(), and logical AND (&&). " + + "Available Bing operators: " + string.Join(", ", s_advancedSearchKeywords)); + } + } + + /// + /// Processes equality and inequality expressions (property == value or property != value). + /// + /// The binary expression to process. + /// The filter to update. + /// True if this is an inequality (!=) expression. + private static void ProcessEqualityExpression(BinaryExpression binaryExpr, TextSearchFilter filter, bool isNegated) + { + // Handle nullable properties with conversions (e.g., bool? == bool becomes Convert(property) == value) + MemberExpression? memberExpr = binaryExpr.Left as MemberExpression; + if (memberExpr == null && binaryExpr.Left is UnaryExpression unaryExpr && unaryExpr.NodeType == ExpressionType.Convert) + { + memberExpr = unaryExpr.Operand as MemberExpression; + } + + // Handle conversions on the right side too + ConstantExpression? constExpr = binaryExpr.Right as ConstantExpression; + if (constExpr == null && binaryExpr.Right is UnaryExpression rightUnaryExpr && rightUnaryExpr.NodeType == ExpressionType.Convert) + { + constExpr = rightUnaryExpr.Operand as ConstantExpression; + } + + if (memberExpr != null && constExpr != null) + { + string propertyName = memberExpr.Member.Name; + object? value = constExpr.Value; + + string? bingFilterName = MapPropertyToBingFilter(propertyName); + if (bingFilterName != null && value != null) + { + // Convert boolean values to lowercase strings for Bing API compatibility + // CA1308: Using ToLowerInvariant() is intentional here as Bing API expects boolean values in lowercase format (true/false) +#pragma warning disable CA1308 // Normalize strings to uppercase + string stringValue = value is bool boolValue ? boolValue.ToString().ToLowerInvariant() : value.ToString() ?? string.Empty; +#pragma warning restore CA1308 // Normalize strings to uppercase + + if (isNegated) + { + // For inequality, wrap the value with a negation marker + // This will be processed in BuildQuery to prepend '-' to the advanced search operator + // Example: language:en becomes -language:en (excludes pages in English) + filter.Equality(bingFilterName, $"-{stringValue}"); + } + else + { + filter.Equality(bingFilterName, stringValue); + } + } + else if (value == null) + { + throw new NotSupportedException( + $"Null values are not supported in Bing API filters for property '{propertyName}'."); + } + else + { + throw new NotSupportedException( + $"Property '{propertyName}' cannot be mapped to Bing API filters. " + + "Supported properties: Language, Url, DisplayUrl, Name, Snippet, IsFamilyFriendly."); + } + } + else + { + throw new NotSupportedException( + "Equality expressions must be in the form 'property == value' or 'property != value'. " + + "Complex expressions on the left or right side are not supported."); + } + } + + /// + /// Processes Contains() method calls on string properties. + /// Maps to Bing's advanced search operators like intitle:, inbody:, url:. + /// + private static void ProcessContainsExpression(MethodCallExpression methodExpr, TextSearchFilter filter) + { + // Contains can be called on a property: page.Name.Contains("value") + // or on a collection: page.Tags.Contains("value") + + if (methodExpr.Object is MemberExpression memberExpr) + { + string propertyName = memberExpr.Member.Name; + + // Extract the search value from the Contains() argument + if (methodExpr.Arguments.Count == 1 && methodExpr.Arguments[0] is ConstantExpression constExpr) + { + object? value = constExpr.Value; + if (value == null) + { + return; // Skip null values + } + + // Map property to Bing filter with Contains semantic + string? bingFilterOperator = MapPropertyToContainsFilter(propertyName); + if (bingFilterOperator != null) + { + // Use Bing's advanced search syntax: intitle:"value", inbody:"value", etc. + filter.Equality(bingFilterOperator, value); + } + else + { + throw new NotSupportedException( + $"Contains() on property '{propertyName}' is not supported by Bing API filters. " + + "Supported properties for Contains: Name (maps to intitle:), Snippet (maps to inbody:), Url (maps to url:)."); + } + } + else + { + throw new NotSupportedException( + "Contains() must have a single constant value argument. " + + "Complex expressions as arguments are not supported."); + } + } + else + { + throw new NotSupportedException( + "Contains() must be called on a property (e.g., page.Name.Contains(\"value\")). " + + "Collection Contains patterns are not yet supported."); + } + } + + /// + /// Maps BingWebPage property names to Bing API filter field names for equality operations. + /// + /// The BingWebPage property name. + /// The corresponding Bing API filter name, or null if not mappable. + private static string? MapPropertyToBingFilter(string propertyName) + { + return propertyName.ToUpperInvariant() switch + { + // Map BingWebPage properties to Bing API equivalents + "LANGUAGE" => "language", // Maps to advanced search + "URL" => "url", // Maps to advanced search + "DISPLAYURL" => "site", // Maps to site: search + "NAME" => "intitle", // Maps to title search + "SNIPPET" => "inbody", // Maps to body content search + "ISFAMILYFRIENDLY" => "safeSearch", // Maps to safe search parameter + + // Direct API parameters (if we ever extend BingWebPage with metadata) + "MKT" => "mkt", // Market/locale + "FRESHNESS" => "freshness", // Date freshness + + _ => null // Property not mappable to Bing filters + }; + } + + /// + /// Maps BingWebPage property names to Bing API advanced search operators for Contains operations. + /// + /// The BingWebPage property name. + /// The corresponding Bing advanced search operator, or null if not mappable. + private static string? MapPropertyToContainsFilter(string propertyName) + { + return propertyName.ToUpperInvariant() switch + { + // Map properties to Bing's contains-style operators + "NAME" => "intitle", // intitle:"search term" - title contains + "SNIPPET" => "inbody", // inbody:"search term" - body contains + "URL" => "url", // url:"search term" - URL contains + "DISPLAYURL" => "site", // site:domain.com - site contains + + _ => null // Property not mappable to Contains-style filters + }; + } + /// /// Execute a Bing search query and return the results. /// @@ -260,14 +525,21 @@ private static string BuildQuery(string query, TextSearchOptions searchOptions) { if (filterClause is EqualToFilterClause equalityFilterClause) { + // Check if value starts with '-' indicating negation (for inequality != operator) + string? valueStr = equalityFilterClause.Value?.ToString(); + bool isNegated = valueStr?.StartsWith("-", StringComparison.Ordinal) == true; + string actualValue = isNegated && valueStr != null ? valueStr.Substring(1) : valueStr ?? string.Empty; + if (s_advancedSearchKeywords.Contains(equalityFilterClause.FieldName, StringComparer.OrdinalIgnoreCase) && equalityFilterClause.Value is not null) { - fullQuery.Append($"+{equalityFilterClause.FieldName}%3A").Append(Uri.EscapeDataString(equalityFilterClause.Value.ToString()!)); + // For advanced search keywords, prepend '-' if negated to exclude results + string prefix = isNegated ? "-" : ""; + fullQuery.Append($"+{prefix}{equalityFilterClause.FieldName}%3A").Append(Uri.EscapeDataString(actualValue)); } else if (s_queryParameters.Contains(equalityFilterClause.FieldName, StringComparer.OrdinalIgnoreCase) && equalityFilterClause.Value is not null) { string? queryParam = s_queryParameters.FirstOrDefault(s => s.Equals(equalityFilterClause.FieldName, StringComparison.OrdinalIgnoreCase)); - queryParams.Append('&').Append(queryParam!).Append('=').Append(Uri.EscapeDataString(equalityFilterClause.Value.ToString()!)); + queryParams.Append('&').Append(queryParam!).Append('=').Append(Uri.EscapeDataString(actualValue)); } else {