diff --git a/dotnet/src/Plugins/Plugins.Web/Brave/BraveTextSearch.cs b/dotnet/src/Plugins/Plugins.Web/Brave/BraveTextSearch.cs index 8fa793ea4efb..40a0044e5a83 100644 --- a/dotnet/src/Plugins/Plugins.Web/Brave/BraveTextSearch.cs +++ b/dotnet/src/Plugins/Plugins.Web/Brave/BraveTextSearch.cs @@ -3,6 +3,7 @@ using System; using System.Collections.Generic; using System.Linq; +using System.Linq.Expressions; using System.Net.Http; using System.Runtime.CompilerServices; using System.Text; @@ -20,7 +21,7 @@ namespace Microsoft.SemanticKernel.Plugins.Web.Brave; /// /// A Brave Text Search implementation that can be used to perform searches using the Brave Web Search API. /// -public sealed class BraveTextSearch : ITextSearch +public sealed class BraveTextSearch : ITextSearch, ITextSearch { /// /// Create an instance of the with API key authentication. @@ -78,7 +79,382 @@ public async Task> GetSearchResultsAsync(string quer return new KernelSearchResults(this.GetResultsAsWebPageAsync(searchResponse, cancellationToken), totalCount, GetResultsMetadata(searchResponse)); } - #region private + #region Generic ITextSearch Implementation + + /// + async Task> ITextSearch.SearchAsync(string query, TextSearchOptions? searchOptions, CancellationToken cancellationToken) + { + var legacyOptions = this.ConvertToLegacyOptions(searchOptions); + BraveSearchResponse? searchResponse = await this.ExecuteSearchAsync(query, legacyOptions, cancellationToken).ConfigureAwait(false); + + long? totalCount = legacyOptions.IncludeTotalCount ? searchResponse?.Web?.Results.Count : null; + + return new KernelSearchResults(this.GetResultsAsStringAsync(searchResponse, cancellationToken), totalCount, GetResultsMetadata(searchResponse)); + } + + /// + async Task> ITextSearch.GetTextSearchResultsAsync(string query, TextSearchOptions? searchOptions, CancellationToken cancellationToken) + { + var legacyOptions = this.ConvertToLegacyOptions(searchOptions); + BraveSearchResponse? searchResponse = await this.ExecuteSearchAsync(query, legacyOptions, cancellationToken).ConfigureAwait(false); + + long? totalCount = legacyOptions.IncludeTotalCount ? searchResponse?.Web?.Results.Count : null; + + return new KernelSearchResults(this.GetResultsAsTextSearchResultAsync(searchResponse, cancellationToken), totalCount, GetResultsMetadata(searchResponse)); + } + + /// + async Task> ITextSearch.GetSearchResultsAsync(string query, TextSearchOptions? searchOptions, CancellationToken cancellationToken) + { + var legacyOptions = this.ConvertToLegacyOptions(searchOptions); + BraveSearchResponse? searchResponse = await this.ExecuteSearchAsync(query, legacyOptions, cancellationToken).ConfigureAwait(false); + + long? totalCount = legacyOptions.IncludeTotalCount ? searchResponse?.Web?.Results.Count : null; + + return new KernelSearchResults(this.GetResultsAsBraveWebPageAsync(searchResponse, cancellationToken), totalCount, GetResultsMetadata(searchResponse)); + } + + #endregion + + #region LINQ-to-Brave Conversion Logic + + /// + /// Converts generic TextSearchOptions with LINQ filtering to legacy TextSearchOptions. + /// + /// The generic search options with LINQ filter. + /// Legacy TextSearchOptions with converted filters. + private TextSearchOptions ConvertToLegacyOptions(TextSearchOptions? options) + { + if (options == null) + { + return new TextSearchOptions(); + } + + var legacyOptions = new TextSearchOptions + { + Top = options.Top, + Skip = options.Skip, + IncludeTotalCount = options.IncludeTotalCount + }; + + // Convert LINQ expression to TextSearchFilter if present + if (options.Filter != null) + { + try + { + var convertedFilter = ConvertLinqExpressionToBraveFilter(options.Filter); + legacyOptions = new TextSearchOptions + { + Top = options.Top, + Skip = options.Skip, + IncludeTotalCount = options.IncludeTotalCount, + Filter = convertedFilter + }; + } + catch (NotSupportedException ex) + { + this._logger.LogWarning("LINQ expression not fully supported by Brave API, performing search without some filters: {Message}", ex.Message); + // Continue with basic search - graceful degradation + } + } + + return legacyOptions; + } + + /// + /// Converts a LINQ expression to Brave-compatible TextSearchFilter. + /// + /// The LINQ expression to convert. + /// A TextSearchFilter with Brave-compatible filter clauses. + private static TextSearchFilter ConvertLinqExpressionToBraveFilter(Expression> linqExpression) + { + var filter = new TextSearchFilter(); + var filterClauses = new List(); + + // Analyze the LINQ expression and convert to filter clauses + AnalyzeExpression(linqExpression.Body, filterClauses); + + // Validate and add clauses that are supported by Brave + foreach (var clause in filterClauses) + { + if (clause is EqualToFilterClause equalityClause) + { + var mappedFieldName = MapPropertyToBraveFilter(equalityClause.FieldName); + if (mappedFieldName != null) + { + filter.Equality(mappedFieldName, equalityClause.Value); + } + else + { + throw new NotSupportedException( + $"Property '{equalityClause.FieldName}' cannot be mapped to Brave API filters. " + + $"Supported properties: {string.Join(", ", s_queryParameters)}. " + + "Example: page => page.Country == \"US\" && page.SafeSearch == \"moderate\""); + } + } + } + + return filter; + } + + /// + /// Maps BraveWebPage property names to Brave API filter parameter names. + /// + /// The property name from BraveWebPage. + /// The corresponding Brave API parameter name, or null if not mappable. + private static string? MapPropertyToBraveFilter(string propertyName) => + propertyName.ToUpperInvariant() switch + { + "COUNTRY" => BraveParamCountry, + "SEARCHLANG" => BraveParamSearchLang, + "UILANG" => BraveParamUiLang, + "SAFESEARCH" => BraveParamSafeSearch, + "TEXTDECORATIONS" => BraveParamTextDecorations, + "SPELLCHECK" => BraveParamSpellCheck, + "RESULTFILTER" => BraveParamResultFilter, + "UNITS" => BraveParamUnits, + "EXTRASNIPPETS" => BraveParamExtraSnippets, + _ => null // Property not mappable to Brave filters + }; + + // TODO: Consider extracting LINQ expression analysis logic to a shared utility class + // to reduce duplication across text search connectors (Brave, Tavily, etc.). + // See code review for details. + /// + /// Analyzes a LINQ expression and extracts filter clauses. + /// + /// The expression to analyze. + /// The list to add extracted filter clauses to. + private static void AnalyzeExpression(Expression expression, List filterClauses) + { + switch (expression) + { + case BinaryExpression binaryExpr: + if (binaryExpr.NodeType == ExpressionType.AndAlso) + { + // Handle AND expressions by recursively analyzing both sides + AnalyzeExpression(binaryExpr.Left, filterClauses); + AnalyzeExpression(binaryExpr.Right, filterClauses); + } + else if (binaryExpr.NodeType == ExpressionType.OrElse) + { + // Handle OR expressions by recursively analyzing both sides + // Note: OR results in multiple filter values for the same property + AnalyzeExpression(binaryExpr.Left, filterClauses); + AnalyzeExpression(binaryExpr.Right, filterClauses); + } + else if (binaryExpr.NodeType == ExpressionType.Equal) + { + // Handle equality expressions + ExtractEqualityClause(binaryExpr, filterClauses); + } + else if (binaryExpr.NodeType == ExpressionType.NotEqual) + { + // Handle inequality expressions (property != value) + // This is supported as a negation pattern + ExtractInequalityClause(binaryExpr, filterClauses); + } + else + { + throw new NotSupportedException($"Binary expression type '{binaryExpr.NodeType}' is not supported. Supported operators: AndAlso (&&), OrElse (||), Equal (==), NotEqual (!=)."); + } + break; + + case UnaryExpression unaryExpr when unaryExpr.NodeType == ExpressionType.Not: + // Handle NOT expressions (negation) + AnalyzeNotExpression(unaryExpr, filterClauses); + break; + + case MethodCallExpression methodCall: + // Handle method calls like Contains, StartsWith, etc. + ExtractMethodCallClause(methodCall, filterClauses); + break; + + default: + throw new NotSupportedException($"Expression type '{expression.NodeType}' is not supported in Brave search filters."); + } + } + + /// + /// Extracts an equality filter clause from a binary equality expression. + /// + /// The binary equality expression. + /// The list to add the extracted clause to. + private static void ExtractEqualityClause(BinaryExpression binaryExpr, List filterClauses) + { + string? propertyName = null; + object? value = null; + + // Determine which side is the property and which is the value + if (binaryExpr.Left is MemberExpression leftMember) + { + propertyName = leftMember.Member.Name; + value = ExtractValue(binaryExpr.Right); + } + else if (binaryExpr.Right is MemberExpression rightMember) + { + propertyName = rightMember.Member.Name; + value = ExtractValue(binaryExpr.Left); + } + + if (propertyName != null && value != null) + { + filterClauses.Add(new EqualToFilterClause(propertyName, value)); + } + else + { + throw new NotSupportedException("Unable to extract property name and value from equality expression."); + } + } + + /// + /// Extracts an inequality filter clause from a binary not-equal expression. + /// + /// The binary not-equal expression. + /// The list to add the extracted clause to. + private static void ExtractInequalityClause(BinaryExpression binaryExpr, List filterClauses) + { + // Note: Inequality is tracked but handled differently depending on the property + // For now, we log a warning that inequality filtering may not work as expected + string? propertyName = null; + object? value = null; + + if (binaryExpr.Left is MemberExpression leftMember) + { + propertyName = leftMember.Member.Name; + value = ExtractValue(binaryExpr.Right); + } + else if (binaryExpr.Right is MemberExpression rightMember) + { + propertyName = rightMember.Member.Name; + value = ExtractValue(binaryExpr.Left); + } + + if (propertyName != null && value != null) + { + // Add a marker for inequality - this will need special handling in conversion + // For now, we don't add it to filter clauses as Brave API doesn't support direct negation + throw new NotSupportedException($"Inequality operator (!=) is not directly supported for property '{propertyName}'. Use NOT operator instead: !(page.{propertyName} == value)."); + } + + throw new NotSupportedException("Unable to extract property name and value from inequality expression."); + } + + /// + /// Analyzes a NOT (negation) expression. + /// + /// The unary NOT expression. + /// The list to add extracted filter clauses to. + private static void AnalyzeNotExpression(UnaryExpression unaryExpr, List filterClauses) + { + // NOT expressions are complex for web search APIs + // We support simple cases like !(page.SafeSearch == "off") + if (unaryExpr.Operand is BinaryExpression binaryExpr && binaryExpr.NodeType == ExpressionType.Equal) + { + // This is !(property == value), which we can handle for some properties + throw new NotSupportedException("NOT operator (!) with equality is not directly supported. Most web search APIs don't support negative filtering."); + } + + throw new NotSupportedException("NOT operator (!) is only supported with simple equality expressions."); + } + + /// + /// Extracts a filter clause from a method call expression (e.g., Contains, StartsWith). + /// + /// The method call expression. + /// The list to add the extracted clause to. + private static void ExtractMethodCallClause(MethodCallExpression methodCall, List filterClauses) + { + if (methodCall.Method.Name == "Contains") + { + // Check if this is property.Contains(value) or array.Contains(property) + if (methodCall.Object is MemberExpression member) + { + // This is property.Contains(value) - e.g., page.ResultFilter.Contains("web") + var propertyName = member.Member.Name; + var value = ExtractValue(methodCall.Arguments[0]); + + if (value != null) + { + // For Contains, we'll map it to equality for certain properties + if (propertyName.Equals("ResultFilter", StringComparison.OrdinalIgnoreCase)) + { + filterClauses.Add(new EqualToFilterClause(propertyName, value)); + } + else + { + throw new NotSupportedException($"Contains method is only supported for ResultFilter property, not '{propertyName}'."); + } + } + } + else if (methodCall.Object == null && methodCall.Arguments.Count == 2) + { + // This is array.Contains(property) - e.g., new[] { "US", "GB" }.Contains(page.Country) + // This is an extension method call where the first argument is the array + var arrayExpr = methodCall.Arguments[0]; + var propertyExpr = methodCall.Arguments[1]; + + if (propertyExpr is MemberExpression propertyMember) + { + var propertyName = propertyMember.Member.Name; + var arrayValue = ExtractValue(arrayExpr); + + if (arrayValue is System.Collections.IEnumerable enumerable) + { + // Convert to OR expressions - each value becomes an equality clause + foreach (var value in enumerable) + { + if (value != null) + { + filterClauses.Add(new EqualToFilterClause(propertyName, value)); + } + } + } + else + { + throw new NotSupportedException($"Contains argument must be an array or collection, got: {arrayValue?.GetType().Name}"); + } + } + else + { + throw new NotSupportedException("Contains with inline collection requires a property reference as the second argument."); + } + } + else + { + throw new NotSupportedException("Unsupported Contains expression format."); + } + } + else + { + throw new NotSupportedException($"Method '{methodCall.Method.Name}' is not supported in Brave search filters. Only 'Contains' is supported."); + } + } + + /// + /// Extracts a constant value from an expression. + /// + /// The expression to extract the value from. + /// The extracted value, or null if extraction failed. + private static object? ExtractValue(Expression expression) + { + return expression switch + { + ConstantExpression constant => constant.Value, + MemberExpression member when member.Expression is ConstantExpression constantExpr => + member.Member switch + { + System.Reflection.FieldInfo field => field.GetValue(constantExpr.Value), + System.Reflection.PropertyInfo property => property.GetValue(constantExpr.Value), + _ => null + }, + _ => Expression.Lambda(expression).Compile().DynamicInvoke() + }; + } + + #endregion + + #region Private Methods private readonly ILogger _logger; private readonly HttpClient _httpClient; @@ -90,8 +466,19 @@ public async Task> GetSearchResultsAsync(string quer private static readonly ITextSearchStringMapper s_defaultStringMapper = new DefaultTextSearchStringMapper(); private static readonly ITextSearchResultMapper s_defaultResultMapper = new DefaultTextSearchResultMapper(); + // Constants for Brave API parameter names + private const string BraveParamCountry = "country"; + private const string BraveParamSearchLang = "search_lang"; + private const string BraveParamUiLang = "ui_lang"; + private const string BraveParamSafeSearch = "safesearch"; + private const string BraveParamTextDecorations = "text_decorations"; + private const string BraveParamSpellCheck = "spellcheck"; + private const string BraveParamResultFilter = "result_filter"; + private const string BraveParamUnits = "units"; + private const string BraveParamExtraSnippets = "extra_snippets"; + // See https://api-dashboard.search.brave.com/app/documentation/web-search/query#WebSearchAPIQueryParameters - private static readonly string[] s_queryParameters = ["country", "search_lang", "ui_lang", "safesearch", "text_decorations", "spellcheck", "result_filter", "units", "extra_snippets"]; + private static readonly string[] s_queryParameters = [BraveParamCountry, BraveParamSearchLang, BraveParamUiLang, BraveParamSafeSearch, BraveParamTextDecorations, BraveParamSpellCheck, BraveParamResultFilter, BraveParamUnits, BraveParamExtraSnippets]; private static readonly string[] s_safeSearch = ["off", "moderate", "strict"]; @@ -178,6 +565,25 @@ private async IAsyncEnumerable GetResultsAsWebPageAsync(BraveSearchRespo } } + /// + /// Return the search results as instances of . + /// + /// Response containing the web pages matching the query. + /// Cancellation token + private async IAsyncEnumerable GetResultsAsBraveWebPageAsync(BraveSearchResponse? searchResponse, [EnumeratorCancellation] CancellationToken cancellationToken) + { + if (searchResponse is null) { yield break; } + + if (searchResponse.Web?.Results is { Count: > 0 } webResults) + { + foreach (var webPage in webResults) + { + yield return BraveWebPage.FromWebResult(webPage); + await Task.Yield(); + } + } + } + /// /// Return the search results as instances of . /// diff --git a/dotnet/src/Plugins/Plugins.Web/Brave/BraveWebPage.cs b/dotnet/src/Plugins/Plugins.Web/Brave/BraveWebPage.cs new file mode 100644 index 000000000000..220d1e3141bb --- /dev/null +++ b/dotnet/src/Plugins/Plugins.Web/Brave/BraveWebPage.cs @@ -0,0 +1,144 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; + +namespace Microsoft.SemanticKernel.Plugins.Web.Brave; + +/// +/// Represents a type-safe web page result from Brave search for use with generic ITextSearch<TRecord> interface. +/// This class provides compile-time type safety and IntelliSense support for Brave search filtering. +/// +public sealed class BraveWebPage +{ + /// + /// Gets or sets the title of the web page. + /// + public string? Title { get; set; } + + /// + /// Gets or sets the URL of the web page. + /// + public string? Url { get; set; } + + /// + /// Gets or sets the description of the web page. + /// + public string? Description { get; set; } + + /// + /// Gets or sets the type of the search result. + /// + public string? Type { get; set; } + + /// + /// Gets or sets the age of the web search result. + /// + public string? Age { get; set; } + + /// + /// Gets or sets the page age timestamp. + /// + public DateTime? PageAge { get; set; } + + /// + /// Gets or sets the language of the web page. + /// + public string? Language { get; set; } + + /// + /// Gets or sets whether the web page is family friendly. + /// + public bool? FamilyFriendly { get; set; } + + /// + /// Gets or sets the country filter for search results. + /// Maps to Brave's 'country' parameter (e.g., "US", "GB", "CA"). + /// + public string? Country { get; set; } + + /// + /// Gets or sets the search language filter. + /// Maps to Brave's 'search_lang' parameter (e.g., "en", "es", "fr"). + /// + public string? SearchLang { get; set; } + + /// + /// Gets or sets the UI language filter. + /// Maps to Brave's 'ui_lang' parameter (e.g., "en-US", "en-GB"). + /// + public string? UiLang { get; set; } + + /// + /// Gets or sets the safe search filter. + /// Maps to Brave's 'safesearch' parameter ("off", "moderate", "strict"). + /// + public string? SafeSearch { get; set; } + + /// + /// Gets or sets whether text decorations are enabled. + /// Maps to Brave's 'text_decorations' parameter. + /// + public bool? TextDecorations { get; set; } + + /// + /// Gets or sets whether spell check is enabled. + /// Maps to Brave's 'spellcheck' parameter. + /// + public bool? SpellCheck { get; set; } + + /// + /// Gets or sets the result filter for search types. + /// Maps to Brave's 'result_filter' parameter (e.g., "web", "news", "videos"). + /// + public string? ResultFilter { get; set; } + + /// + /// Gets or sets the units system for measurements. + /// Maps to Brave's 'units' parameter ("metric" or "imperial"). + /// + public string? Units { get; set; } + + /// + /// Gets or sets whether extra snippets are included. + /// Maps to Brave's 'extra_snippets' parameter. + /// + public bool? ExtraSnippets { get; set; } + + /// + /// Initializes a new instance of the class. + /// + public BraveWebPage() + { + } + + /// + /// Initializes a new instance of the class with specified values. + /// + /// The title of the web page. + /// The URL of the web page. + /// The description of the web page. + /// The type of the search result. + public BraveWebPage(string? title, string? url, string? description, string? type = null) + { + this.Title = title; + this.Url = url; + this.Description = description; + this.Type = type; + } + + /// + /// Creates a BraveWebPage from a BraveWebResult. + /// + /// The web result to convert. + /// A new BraveWebPage instance. + internal static BraveWebPage FromWebResult(BraveWebResult result) + { + return new BraveWebPage(result.Title, result.Url, result.Description, result.Type) + { + Age = result.Age, + PageAge = result.PageAge, + Language = result.Language, + FamilyFriendly = result.FamilyFriendly + }; + } +} diff --git a/dotnet/src/Plugins/Plugins.Web/Tavily/TavilyTextSearch.cs b/dotnet/src/Plugins/Plugins.Web/Tavily/TavilyTextSearch.cs index 4e01d0ffb88b..e82d72669505 100644 --- a/dotnet/src/Plugins/Plugins.Web/Tavily/TavilyTextSearch.cs +++ b/dotnet/src/Plugins/Plugins.Web/Tavily/TavilyTextSearch.cs @@ -2,6 +2,7 @@ using System; using System.Collections.Generic; +using System.Linq.Expressions; using System.Net.Http; using System.Runtime.CompilerServices; using System.Text; @@ -20,7 +21,7 @@ namespace Microsoft.SemanticKernel.Plugins.Web.Tavily; /// /// A Tavily Text Search implementation that can be used to perform searches using the Tavily Web Search API. /// -public sealed class TavilyTextSearch : ITextSearch +public sealed class TavilyTextSearch : ITextSearch, ITextSearch { /// /// Create an instance of the with API key authentication. @@ -75,7 +76,378 @@ public async Task> GetSearchResultsAsync(string quer return new KernelSearchResults(this.GetSearchResultsAsync(searchResponse, cancellationToken), totalCount, GetResultsMetadata(searchResponse)); } - #region private + #region Generic ITextSearch Implementation + + /// + async Task> ITextSearch.SearchAsync(string query, TextSearchOptions? searchOptions, CancellationToken cancellationToken) + { + var legacyOptions = this.ConvertToLegacyOptions(searchOptions); + TavilySearchResponse? searchResponse = await this.ExecuteSearchAsync(query, legacyOptions, cancellationToken).ConfigureAwait(false); + + long? totalCount = null; + + return new KernelSearchResults(this.GetResultsAsStringAsync(searchResponse, cancellationToken), totalCount, GetResultsMetadata(searchResponse)); + } + + /// + async Task> ITextSearch.GetTextSearchResultsAsync(string query, TextSearchOptions? searchOptions, CancellationToken cancellationToken) + { + var legacyOptions = this.ConvertToLegacyOptions(searchOptions); + TavilySearchResponse? searchResponse = await this.ExecuteSearchAsync(query, legacyOptions, cancellationToken).ConfigureAwait(false); + + long? totalCount = null; + + return new KernelSearchResults(this.GetResultsAsTextSearchResultAsync(searchResponse, cancellationToken), totalCount, GetResultsMetadata(searchResponse)); + } + + /// + async Task> ITextSearch.GetSearchResultsAsync(string query, TextSearchOptions? searchOptions, CancellationToken cancellationToken) + { + var legacyOptions = this.ConvertToLegacyOptions(searchOptions); + TavilySearchResponse? searchResponse = await this.ExecuteSearchAsync(query, legacyOptions, cancellationToken).ConfigureAwait(false); + + long? totalCount = null; + + return new KernelSearchResults(this.GetResultsAsWebPageAsync(searchResponse, cancellationToken), totalCount, GetResultsMetadata(searchResponse)); + } + + #endregion + + #region LINQ-to-Tavily Conversion Logic + + /// + /// Converts generic TextSearchOptions with LINQ filtering to legacy TextSearchOptions. + /// + /// The generic search options with LINQ filter. + /// Legacy TextSearchOptions with converted filters. + private TextSearchOptions ConvertToLegacyOptions(TextSearchOptions? options) + { + if (options == null) + { + return new TextSearchOptions(); + } + + var legacyOptions = new TextSearchOptions + { + Top = options.Top, + Skip = options.Skip, + IncludeTotalCount = options.IncludeTotalCount + }; + + // Convert LINQ expression to TextSearchFilter if present + if (options.Filter != null) + { + try + { + var convertedFilter = ConvertLinqExpressionToTavilyFilter(options.Filter); + legacyOptions = new TextSearchOptions + { + Top = options.Top, + Skip = options.Skip, + IncludeTotalCount = options.IncludeTotalCount, + Filter = convertedFilter + }; + } + catch (NotSupportedException ex) + { + this._logger.LogWarning("LINQ expression not fully supported by Tavily API, performing search without some filters: {Message}", ex.Message); + // Continue with basic search - graceful degradation + } + } + + return legacyOptions; + } + + /// + /// Converts a LINQ expression to Tavily-compatible TextSearchFilter. + /// + /// The LINQ expression to convert. + /// A TextSearchFilter with Tavily-compatible filter clauses. + private static TextSearchFilter ConvertLinqExpressionToTavilyFilter(Expression> linqExpression) + { + var filter = new TextSearchFilter(); + var filterClauses = new List(); + + // Analyze the LINQ expression and convert to filter clauses + AnalyzeExpression(linqExpression.Body, filterClauses); + + // Validate and add clauses that are supported by Tavily + foreach (var clause in filterClauses) + { + if (clause is EqualToFilterClause equalityClause) + { + var mappedFieldName = MapPropertyToTavilyFilter(equalityClause.FieldName); + if (mappedFieldName != null) + { + filter.Equality(mappedFieldName, equalityClause.Value); + } + else + { + throw new NotSupportedException( + $"Property '{equalityClause.FieldName}' cannot be mapped to Tavily API filters. " + + $"Supported properties: {string.Join(", ", s_validFieldNames)}. " + + "Example: page => page.Topic == \"general\" && page.TimeRange == \"week\""); + } + } + } + + return filter; + } + + /// + /// Maps TavilyWebPage property names to Tavily API filter parameter names. + /// + /// The property name from TavilyWebPage. + /// The corresponding Tavily API parameter name, or null if not mappable. + private static string? MapPropertyToTavilyFilter(string propertyName) => + propertyName.ToUpperInvariant() switch + { + "TOPIC" => Topic, + "TIMERANGE" => TimeRange, + "DAYS" => Days, + "INCLUDEDOMAIN" => IncludeDomain, + "EXCLUDEDOMAIN" => ExcludeDomain, + _ => null // Property not mappable to Tavily filters + }; + + // TODO: Consider extracting LINQ expression analysis logic to a shared utility class + // to reduce duplication across text search connectors (Brave, Tavily, etc.). + // See code review for details. + /// + /// Analyzes a LINQ expression and extracts filter clauses. + /// + /// The expression to analyze. + /// The list to add extracted filter clauses to. + private static void AnalyzeExpression(Expression expression, List filterClauses) + { + switch (expression) + { + case BinaryExpression binaryExpr: + if (binaryExpr.NodeType == ExpressionType.AndAlso) + { + // Handle AND expressions by recursively analyzing both sides + AnalyzeExpression(binaryExpr.Left, filterClauses); + AnalyzeExpression(binaryExpr.Right, filterClauses); + } + else if (binaryExpr.NodeType == ExpressionType.OrElse) + { + // Handle OR expressions by recursively analyzing both sides + // Note: OR results in multiple filter values for the same property (especially for domains) + AnalyzeExpression(binaryExpr.Left, filterClauses); + AnalyzeExpression(binaryExpr.Right, filterClauses); + } + else if (binaryExpr.NodeType == ExpressionType.Equal) + { + // Handle equality expressions + ExtractEqualityClause(binaryExpr, filterClauses); + } + else if (binaryExpr.NodeType == ExpressionType.NotEqual) + { + // Handle inequality expressions (property != value) + // This is supported as a negation pattern + ExtractInequalityClause(binaryExpr, filterClauses); + } + else + { + throw new NotSupportedException($"Binary expression type '{binaryExpr.NodeType}' is not supported. Supported operators: AndAlso (&&), OrElse (||), Equal (==), NotEqual (!=)."); + } + break; + + case UnaryExpression unaryExpr when unaryExpr.NodeType == ExpressionType.Not: + // Handle NOT expressions (negation) + AnalyzeNotExpression(unaryExpr, filterClauses); + break; + + case MethodCallExpression methodCall: + // Handle method calls like Contains, StartsWith, etc. + ExtractMethodCallClause(methodCall, filterClauses); + break; + + default: + throw new NotSupportedException($"Expression type '{expression.NodeType}' is not supported in Tavily search filters."); + } + } + + /// + /// Extracts an equality filter clause from a binary equality expression. + /// + /// The binary equality expression. + /// The list to add the extracted clause to. + private static void ExtractEqualityClause(BinaryExpression binaryExpr, List filterClauses) + { + string? propertyName = null; + object? value = null; + + // Determine which side is the property and which is the value + if (binaryExpr.Left is MemberExpression leftMember) + { + propertyName = leftMember.Member.Name; + value = ExtractValue(binaryExpr.Right); + } + else if (binaryExpr.Right is MemberExpression rightMember) + { + propertyName = rightMember.Member.Name; + value = ExtractValue(binaryExpr.Left); + } + + if (propertyName != null && value != null) + { + filterClauses.Add(new EqualToFilterClause(propertyName, value)); + } + else + { + throw new NotSupportedException("Unable to extract property name and value from equality expression."); + } + } + + /// + /// Extracts an inequality filter clause from a binary not-equal expression. + /// + /// The binary not-equal expression. + /// The list to add the extracted clause to. + private static void ExtractInequalityClause(BinaryExpression binaryExpr, List filterClauses) + { + // Note: Inequality is tracked but handled differently depending on the property + // For now, we log a warning that inequality filtering may not work as expected + string? propertyName = null; + object? value = null; + + if (binaryExpr.Left is MemberExpression leftMember) + { + propertyName = leftMember.Member.Name; + value = ExtractValue(binaryExpr.Right); + } + else if (binaryExpr.Right is MemberExpression rightMember) + { + propertyName = rightMember.Member.Name; + value = ExtractValue(binaryExpr.Left); + } + + if (propertyName != null && value != null) + { + // Add a marker for inequality - this will need special handling in conversion + // For now, we don't add it to filter clauses as Tavily API doesn't support direct negation + throw new NotSupportedException($"Inequality operator (!=) is not directly supported for property '{propertyName}'. Use NOT operator instead: !(page.{propertyName} == value)."); + } + + throw new NotSupportedException("Unable to extract property name and value from inequality expression."); + } + + /// + /// Analyzes a NOT (negation) expression. + /// + /// The unary NOT expression. + /// The list to add extracted filter clauses to. + private static void AnalyzeNotExpression(UnaryExpression unaryExpr, List filterClauses) + { + // NOT expressions are complex for web search APIs + // We support simple cases like !(page.Topic == "general") + if (unaryExpr.Operand is BinaryExpression binaryExpr && binaryExpr.NodeType == ExpressionType.Equal) + { + // This is !(property == value), which we can handle for some properties + throw new NotSupportedException("NOT operator (!) with equality is not directly supported. Most web search APIs don't support negative filtering."); + } + + throw new NotSupportedException("NOT operator (!) is only supported with simple equality expressions."); + } + + /// + /// Extracts a filter clause from a method call expression (e.g., Contains, StartsWith). + /// + /// The method call expression. + /// The list to add the extracted clause to. + private static void ExtractMethodCallClause(MethodCallExpression methodCall, List filterClauses) + { + if (methodCall.Method.Name == "Contains") + { + // Check if this is property.Contains(value) or array.Contains(property) + if (methodCall.Object is MemberExpression member) + { + // This is property.Contains(value) - e.g., page.IncludeDomain.Contains("wikipedia.org") + var propertyName = member.Member.Name; + var value = ExtractValue(methodCall.Arguments[0]); + + if (value != null) + { + // For Contains, we'll map it to equality for domains (Tavily supports domain filtering) + if (propertyName.EndsWith("Domain", StringComparison.OrdinalIgnoreCase)) + { + filterClauses.Add(new EqualToFilterClause(propertyName, value)); + } + else + { + throw new NotSupportedException($"Contains method is only supported for domain properties (IncludeDomain, ExcludeDomain), not '{propertyName}'."); + } + } + } + else if (methodCall.Object == null && methodCall.Arguments.Count == 2) + { + // This is array.Contains(property) - e.g., new[] { "general", "news" }.Contains(page.Topic) + // This is an extension method call where the first argument is the array + var arrayExpr = methodCall.Arguments[0]; + var propertyExpr = methodCall.Arguments[1]; + + if (propertyExpr is MemberExpression propertyMember) + { + var propertyName = propertyMember.Member.Name; + var arrayValue = ExtractValue(arrayExpr); + + if (arrayValue is System.Collections.IEnumerable enumerable) + { + // Convert to OR expressions - each value becomes an equality clause + foreach (var value in enumerable) + { + if (value != null) + { + filterClauses.Add(new EqualToFilterClause(propertyName, value)); + } + } + } + else + { + throw new NotSupportedException($"Contains argument must be an array or collection, got: {arrayValue?.GetType().Name}"); + } + } + else + { + throw new NotSupportedException("Contains with inline collection requires a property reference as the second argument."); + } + } + else + { + throw new NotSupportedException("Unsupported Contains expression format."); + } + } + else + { + throw new NotSupportedException($"Method '{methodCall.Method.Name}' is not supported in Tavily search filters. Only 'Contains' is supported."); + } + } + + /// + /// Extracts a constant value from an expression. + /// + /// The expression to extract the value from. + /// The extracted value, or null if extraction failed. + private static object? ExtractValue(Expression expression) + { + return expression switch + { + ConstantExpression constant => constant.Value, + MemberExpression member when member.Expression is ConstantExpression constantExpr => + member.Member switch + { + System.Reflection.FieldInfo field => field.GetValue(constantExpr.Value), + System.Reflection.PropertyInfo property => property.GetValue(constantExpr.Value), + _ => null + }, + _ => Expression.Lambda(expression).Compile().DynamicInvoke() + }; + } + + #endregion + + #region Private Methods private readonly ILogger _logger; private readonly HttpClient _httpClient; @@ -175,6 +547,40 @@ private async IAsyncEnumerable GetSearchResultsAsync(TavilySearchRespons } } + /// + /// Return the search results as instances of . + /// + /// Response containing the web pages matching the query. + /// Cancellation token + private async IAsyncEnumerable GetResultsAsWebPageAsync(TavilySearchResponse? searchResponse, [EnumeratorCancellation] CancellationToken cancellationToken) + { + if (searchResponse is null || searchResponse.Results is null) + { + yield break; + } + + foreach (var result in searchResponse.Results) + { + yield return TavilyWebPage.FromSearchResult(result); + await Task.Yield(); + } + + if (this._searchOptions?.IncludeImages ?? false && searchResponse.Images is not null) + { + foreach (var image in searchResponse.Images!) + { + // For images, create a basic TavilyWebPage representation + yield return new TavilyWebPage( + title: "Image Result", + url: image.Url, + content: image.Description ?? string.Empty, + score: 0.0 + ); + await Task.Yield(); + } + } + } + /// /// Return the search results as instances of . /// diff --git a/dotnet/src/Plugins/Plugins.Web/Tavily/TavilyWebPage.cs b/dotnet/src/Plugins/Plugins.Web/Tavily/TavilyWebPage.cs new file mode 100644 index 000000000000..1c1b45578fce --- /dev/null +++ b/dotnet/src/Plugins/Plugins.Web/Tavily/TavilyWebPage.cs @@ -0,0 +1,99 @@ +// Copyright (c) Microsoft. All rights reserved. + +namespace Microsoft.SemanticKernel.Plugins.Web.Tavily; + +/// +/// Represents a type-safe web page result from Tavily search for use with generic ITextSearch<TRecord> interface. +/// This class provides compile-time type safety and IntelliSense support for Tavily search filtering. +/// +public sealed class TavilyWebPage +{ + /// + /// Gets or sets the title of the web page. + /// + public string? Title { get; set; } + + /// + /// Gets or sets the URL of the web page. + /// + public string? Url { get; set; } + + /// + /// Gets or sets the content/description of the web page. + /// + public string? Content { get; set; } + + /// + /// Gets or sets the raw content of the web page (if available). + /// + public string? RawContent { get; set; } + + /// + /// Gets or sets the relevance score of the search result. + /// + public double Score { get; set; } + + /// + /// Gets or sets the topic filter for search results. + /// Maps to Tavily's 'topic' parameter for focused search. + /// + public string? Topic { get; set; } + + /// + /// Gets or sets the time range filter for search results. + /// Maps to Tavily's 'time_range' parameter (e.g., "day", "week", "month", "year"). + /// + public string? TimeRange { get; set; } + + /// + /// Gets or sets the number of days for time-based filtering. + /// Maps to Tavily's 'days' parameter for custom date ranges. + /// + public int? Days { get; set; } + + /// + /// Gets or sets the domain to include in search results. + /// Maps to Tavily's 'include_domain' parameter. + /// + public string? IncludeDomain { get; set; } + + /// + /// Gets or sets the domain to exclude from search results. + /// Maps to Tavily's 'exclude_domain' parameter. + /// + public string? ExcludeDomain { get; set; } + + /// + /// Initializes a new instance of the class. + /// + public TavilyWebPage() + { + } + + /// + /// Initializes a new instance of the class with specified values. + /// + /// The title of the web page. + /// The URL of the web page. + /// The content/description of the web page. + /// The relevance score. + /// The raw content (optional). + public TavilyWebPage(string? title, string? url, string? content, double score, string? rawContent = null) + { + this.Title = title; + this.Url = url; + this.Content = content; + this.Score = score; + this.RawContent = rawContent; + } + + /// + /// Creates a TavilyWebPage from a TavilySearchResult. + /// + /// The search result to convert. + /// A new TavilyWebPage instance. + internal static TavilyWebPage FromSearchResult(TavilySearchResult result) + { + return new TavilyWebPage(result.Title, result.Url, result.Content, result.Score, result.RawContent); + } +}