(#199) async OData query support in the table controller.

Author: Adrian Hall

src/CommunityToolkit.Datasync.Server.Abstractions/Tables/IRepository.cs

@@ -66,4 +66,24 @@ public interface IRepository<TEntity> where TEntity : ITableData
     /// <exception cref="HttpException">Thrown if the entity creation would produce a normal HTTP error.</exception>
     /// <exception cref="RepositoryException">Thrown is there is an error in the repository.</exception>
     ValueTask ReplaceAsync(TEntity entity, byte[]? version = null, CancellationToken cancellationToken = default);
+
+    /// <summary>
+    /// A method that is used to count the number of entities in an <see cref="IQueryable{T}"/> query using the
+    /// preferred mechanism for the repository.
+    /// </summary>
+    /// <param name="query">The query to be executed.</param>
+    /// <param name="cancellationToken">A <see cref="CancellationToken"/> to observe.</param>
+    /// <returns>The number of items matching the query.</returns>
+    ValueTask<int> CountAsync(IQueryable<TEntity> query, CancellationToken cancellationToken = default)
+        => ValueTask.FromResult(query.Count());
+
+    /// <summary>
+    /// A method that is used to execute the query and return the elements matches the query as a list using the
+    /// preferred mechanism for the repository.
+    /// </summary>
+    /// <param name="query">The query to be executed.</param>
+    /// <param name="cancellationToken">A <see cref="CancellationToken"/> to observe.</param>
+    /// <returns>The items matching the query.</returns>
+    ValueTask<IList<TEntity>> ToListAsync(IQueryable<TEntity> query, CancellationToken cancellationToken = default)
+        => ValueTask.FromResult<IList<TEntity>>(query.ToList());
 }

src/CommunityToolkit.Datasync.Server/Controllers/TableController.Query.cs

@@ -16,7 +16,6 @@
 using Microsoft.OData;
 using System.Diagnostics.CodeAnalysis;
 using CommunityToolkit.Datasync.Server.OData;
-using Microsoft.AspNetCore.Http.Extensions;
 
 namespace CommunityToolkit.Datasync.Server;
 
@@ -38,6 +37,9 @@ public partial class TableController<TEntity> : ODataController where TEntity :
     /// - <c>$skip</c> is used to skip some entities
     /// - <c>$top</c> is used to limit the number of entities returned.
     /// </para>
+    /// <para>
+    /// In addition, the <c>__includeDeleted</c> parameter is used to decide whether to include soft-deleted items in the result.
+    /// </para>
     /// </summary>
     /// <param name="cancellationToken">A cancellation token</param>
     /// <returns>An <see cref="OkObjectResult"/> response object with the items.</returns>
@@ -50,10 +52,6 @@ public virtual async Task<IActionResult> QueryAsync(CancellationToken cancellati
         await AuthorizeRequestAsync(TableOperation.Query, null, cancellationToken).ConfigureAwait(false);
         _ = BuildServiceProvider(Request);
 
-        IQueryable<TEntity> dataset = (await Repository.AsQueryableAsync(cancellationToken).ConfigureAwait(false))
-            .ApplyDataView(AccessControlProvider.GetDataView())
-            .ApplyDeletedView(Request, Options.EnableSoftDelete);
-
         ODataValidationSettings validationSettings = new() { MaxTop = Options.MaxTop };
         ODataQuerySettings querySettings = new() { PageSize = Options.PageSize, EnsureStableOrdering = true };
         ODataQueryContext queryContext = new(EdmModel, typeof(TEntity), new ODataPath());
@@ -69,26 +67,28 @@ public virtual async Task<IActionResult> QueryAsync(CancellationToken cancellati
             return BadRequest(validationException.Message);
         }
 
-        // Note that some IQueryable providers cannot execute all queries against the data source, so we have
-        // to switch to in-memory processing for those queries.  This is done by calling ToListAsync() on the
-        // IQueryable.  This is not ideal, but it is the only way to support all of the OData query options.
-        IEnumerable<object>? results = null;
-        await ExecuteQueryWithClientEvaluationAsync(dataset, ds =>
-        {
-            results = (IEnumerable<object>)queryOptions.ApplyTo(ds, querySettings);
-            return Task.CompletedTask;
-        });
+        // Determine the dataset to be queried for this user.
+        IQueryable<TEntity> dataset = (await Repository.AsQueryableAsync(cancellationToken).ConfigureAwait(false))
+            .ApplyDataView(AccessControlProvider.GetDataView())
+            .ApplyDeletedView(Request, Options.EnableSoftDelete);
 
-        int count = 0;
-        FilterQueryOption? filter = queryOptions.Filter;
-        await ExecuteQueryWithClientEvaluationAsync(dataset, async ds => 
-        { 
-            IQueryable<TEntity> q = (IQueryable<TEntity>)(filter?.ApplyTo(ds, new ODataQuerySettings()) ?? ds);
-            count = await CountAsync(q, cancellationToken);
-        });
+        // Apply the requested filter from the OData transaction.
+        IQueryable<TEntity> filteredDataset = dataset.ApplyODataFilter(queryOptions.Filter, querySettings);
+
+        // Count the number of items within the filtered dataset - this is used when $count is requested.
+        int filteredCount = await Repository.CountAsync(filteredDataset, cancellationToken).ConfigureAwait(false);
+
+        // Now apply the OrderBy, Skip, and Top options to the dataset.
+        IQueryable<TEntity> orderedDataset = filteredDataset
+            .ApplyODataOrderBy(queryOptions.OrderBy, querySettings)
+            .ApplyODataPaging(queryOptions, querySettings);
+
+        // Get the list of items within the dataset that need to be returned.
+        IList<TEntity> entitiesInResultSet = await Repository.ToListAsync(orderedDataset, cancellationToken).ConfigureAwait(false);
 
-        PagedResult result = BuildPagedResult(queryOptions, results, count);
-        Logger.LogInformation("Query: {Count} items being returned", result.Items.Count());
+        // Produce the paged result.
+        PagedResult result = BuildPagedResult(queryOptions, entitiesInResultSet.ApplyODataSelect(queryOptions.SelectExpand, querySettings), filteredCount);
+        Logger.LogInformation("Query: {Count} items being returned", entitiesInResultSet.Count);
         return Ok(result);
     }
 
@@ -135,136 +135,39 @@ internal PagedResult BuildPagedResult(ODataQueryOptions queryOptions, IEnumerabl
         int resultCount = results?.Count() ?? 0;
         int skip = (queryOptions.Skip?.Value ?? 0) + resultCount;
         int top = (queryOptions.Top?.Value ?? 0) - resultCount;
-        if (results is IEnumerable<ISelectExpandWrapper> wrapper)
-        {
-            results = wrapper.Select(x => x.ToDictionary());
-        }
 
-        PagedResult result = new(results ?? []) { Count = queryOptions.Count != null ? count : null };
-        if (queryOptions.Top != null)
-        {
-            result.NextLink = skip >= count || top <= 0 ? null : CreateNextLink(Request, skip, top);
-        }
-        else
+        // Internal function to create the nextLink property for the paged result.
+        static string CreateNextLink(HttpRequest request, int skip = 0, int top = 0)
         {
-            result.NextLink = skip >= count ? null : CreateNextLink(Request, skip, 0);
-        }
+            string? queryString = request.QueryString.Value;
+            List<string> query = (queryString ?? "").TrimStart('?')
+                .Split('&')
+                .Where(q => !q.StartsWith($"{SkipParameterName}=") && !q.StartsWith($"{TopParameterName}="))
+                .ToList();
 
-        return result;
-    }
-
-    /// <summary>
-    /// Given a very specific URI, creates a new query string with the same query, but with a different value for the <c>$skip</c> parameter.
-    /// </summary>
-    /// <param name="request">The original request.</param>
-    /// <param name="skip">The new skip value.</param>
-    /// <param name="top">The new top value.</param>
-    /// <returns>The new URI for the next page of items.</returns>
-    [NonAction]
-    [SuppressMessage("Roslynator", "RCS1158:Static member in generic type should use a type parameter", Justification = "Static method in generic non-static class")]
-    internal static string CreateNextLink(HttpRequest request, int skip = 0, int top = 0)
-        => CreateNextLink(new UriBuilder(request.GetDisplayUrl()).Query, skip, top);
-
-    /// <summary>
-    /// Given a very specific query string,  creates a new query string with the same query, but with a different value for the <c>$skip</c> parameter.
-    /// </summary>
-    /// <param name="queryString">The original query string.</param>
-    /// <param name="skip">The new skip value.</param>
-    /// <param name="top">The new top value.</param>
-    /// <returns>The new URI for the next page of items.</returns>
-    [NonAction]
-    [SuppressMessage("Roslynator", "RCS1158:Static member in generic type should use a type parameter", Justification = "Static method in generic non-static class")]
-    internal static string CreateNextLink(string queryString, int skip = 0, int top = 0)
-    {
-        List<string> query = (queryString ?? "").TrimStart('?')
-            .Split('&')
-            .Where(q => !q.StartsWith($"{SkipParameterName}=") && !q.StartsWith($"{TopParameterName}="))
-            .ToList();
-
-        if (skip > 0)
-        {
-            query.Add($"{SkipParameterName}={skip}");
-        }
-
-        if (top > 0)
-        {
-            query.Add($"{TopParameterName}={top}");
-        }
-
-        return string.Join('&', query).TrimStart('&');
-    }
-
-    /// <summary>
-    /// When doing a query evaluation, certain providers (e.g. Entity Framework) require some things
-    /// to be done client side.  We use a client side evaluator to handle this case when it happens.
-    /// </summary>
-    /// <param name="ex">The exception thrown by the service-side evaluator</param>
-    /// <param name="reason">The reason if the client-side evaluator throws.</param>
-    /// <param name="clientSideEvaluator">The client-side evaluator</param>
-    [NonAction]
-    internal async Task CatchClientSideEvaluationExceptionAsync(Exception ex, string reason, Func<Task> clientSideEvaluator)
-    {
-        if (IsClientSideEvaluationException(ex) || IsClientSideEvaluationException(ex.InnerException))
-        {
-            try
+            if (skip > 0)
             {
-                await clientSideEvaluator.Invoke();
+                query.Add($"{SkipParameterName}={skip}");
             }
-            catch (Exception err)
+
+            if (top > 0)
             {
-                Logger.LogError("Error while {reason}: {Message}", reason, err.Message);
-                throw;
+                query.Add($"{TopParameterName}={top}");
             }
+
+            return string.Join('&', query).TrimStart('&');
         }
-        else
-        {
-            throw ex;
-        }
-    }
 
-    /// <summary>
-    /// Executes an evaluation of a query, using a client-side evaluation if necessary.
-    /// </summary>
-    /// <param name="dataset">The dataset to be evaluated.</param>
-    /// <param name="evaluator">The base evaluation to be performed.</param>
-    [NonAction]
-    internal async Task ExecuteQueryWithClientEvaluationAsync(IQueryable<TEntity> dataset, Func<IQueryable<TEntity>, Task> evaluator)
-    {
-        try
+        PagedResult result = new(results ?? []) { Count = queryOptions.Count != null ? count : null };
+        if (queryOptions.Top is not null)
         {
-            await evaluator.Invoke(dataset);
+            result.NextLink = skip >= count || top <= 0 ? null : CreateNextLink(Request, skip, top);
         }
-        catch (Exception ex) when (!Options.DisableClientSideEvaluation)
+        else
         {
-            await CatchClientSideEvaluationExceptionAsync(ex, "executing query", async () =>
-            {
-                Logger.LogWarning("Error while executing query: possible client-side evaluation ({Message})", ex.InnerException?.Message ?? ex.Message);
-                await evaluator.Invoke(dataset.ToList().AsQueryable());
-            });
+            result.NextLink = skip >= count ? null : CreateNextLink(Request, skip, 0);
         }
-    }
 
-    /// <summary>
-    /// Determines if a particular exception indicates a client-side evaluation is required.
-    /// </summary>
-    /// <param name="ex">The exception that was thrown by the service-side evaluator</param>
-    /// <returns>true if a client-side evaluation is required.</returns>
-    [NonAction]
-    [SuppressMessage("Roslynator", "RCS1158:Static member in generic type should use a type parameter.")]
-    internal static bool IsClientSideEvaluationException(Exception? ex)
-        => ex is not null and (InvalidOperationException or NotSupportedException);
-
-    /// <summary>
-    /// This is an overridable method that calls Count() on the provided queryable.  You can override
-    /// this to calls a provider-specific count mechanism (e.g. CountAsync().
-    /// </summary>
-    /// <param name="query"></param>
-    /// <param name="cancellationToken"></param>
-    /// <returns></returns>
-    [NonAction]
-    public virtual Task<int> CountAsync(IQueryable<TEntity> query, CancellationToken cancellationToken)
-    {
-        int result = query.Count();
-        return Task.FromResult(result);
+        return result;
     }
 }

src/CommunityToolkit.Datasync.Server/Extensions/InternalExtensions.cs

@@ -5,9 +5,12 @@
 using CommunityToolkit.Datasync.Server.Abstractions.Json;
 using Microsoft.AspNetCore.Http;
 using Microsoft.AspNetCore.Http.Headers;
+using Microsoft.AspNetCore.OData.Query;
+using Microsoft.AspNetCore.OData.Query.Wrapper;
 using Microsoft.Extensions.Primitives;
 using Microsoft.Net.Http.Headers;
 using System.Globalization;
+using System.Linq;
 using System.Linq.Expressions;
 using System.Text.Json;
 using System.Text.Json.Serialization;
@@ -35,14 +38,90 @@ internal static IQueryable<T> ApplyDataView<T>(this IQueryable<T> query, Express
     /// <summary>
     /// Filters out the deleted entities unless the request includes an optional parameter to include them.
     /// </summary>
-    /// <typeparam name="T">The type of entity being queries.</typeparam>
+    /// <typeparam name="T">The type of entity being queried.</typeparam>
     /// <param name="query">The current <see cref="IQueryable{T}"/> representing the query.</param>
     /// <param name="request">The current <see cref="HttpRequest"/> being processed.</param>
     /// <param name="enableSoftDelete">A flag to indicate if soft-delete is enabled on the table being queried.</param>
     /// <returns>An updated <see cref="IQueryable{T}"/> representing the new query.</returns>
     internal static IQueryable<T> ApplyDeletedView<T>(this IQueryable<T> query, HttpRequest request, bool enableSoftDelete) where T : ITableData
         => !enableSoftDelete || request.ShouldIncludeDeletedEntities() ? query : query.Where(e => !e.Deleted);
 
+    /// <summary>
+    /// Applies the <c>$filter</c> OData query option to the provided query.
+    /// </summary>
+    /// <typeparam name="T">The type of entity being queried.</typeparam>
+    /// <param name="query">The current <see cref="IQueryable{T}"/> representing the query.</param>
+    /// <param name="filterQueryOption">The filter query option to apply.</param>
+    /// <param name="settings">The query settings being used.</param>
+    /// <returns>A modified <see cref="IQueryable{T}"/> representing the filtered data.</returns>
+    internal static IQueryable<T> ApplyODataFilter<T>(this IQueryable<T> query, FilterQueryOption? filterQueryOption, ODataQuerySettings settings)
+    {
+        if (filterQueryOption is null)
+        {
+            return query;
+        }
+
+        return ((IQueryable<T>?)filterQueryOption.ApplyTo(query, settings)) ?? query;
+    }
+
+    /// <summary>
+    /// Applies the <c>$orderBy</c> OData query option to the provided query.
+    /// </summary>
+    /// <typeparam name="T">The type of entity being queried.</typeparam>
+    /// <param name="query">The current <see cref="IQueryable{T}"/> representing the query.</param>
+    /// <param name="orderingQueryOption">The ordering query option to apply.</param>
+    /// <param name="settings">The query settings being used.</param>
+    /// <returns>A modified <see cref="IQueryable{T}"/> representing the ordered data.</returns>
+    internal static IQueryable<T> ApplyODataOrderBy<T>(this IQueryable<T> query, OrderByQueryOption? orderingQueryOption, ODataQuerySettings settings) where T : ITableData
+    {
+        // Note that we ALWAYS do a sort so that the ordering is consistent across all queries.
+        if (orderingQueryOption is null)
+        {
+            return query.OrderBy(e => e.Id);
+        }
+
+        IOrderedQueryable<T>? orderedQuery = orderingQueryOption.ApplyTo(query, settings);
+        if (orderedQuery is null)
+        {
+            return query.OrderBy(e => e.Id);
+        }
+
+        return orderedQuery.ThenBy(x => x.Id);
+    }
+
+    /// <summary>
+    /// Applies the <c>$skip</c> and <c>$top</c> OData query options to the provided query.
+    /// </summary>
+    /// <typeparam name="T">The type of entity being queried.</typeparam>
+    /// <param name="query">The current <see cref="IQueryable{T}"/> representing the query.</param>
+    /// <param name="options">The query options to apply.</param>
+    /// <param name="settings">The query settings being used.</param>
+    /// <returns>A modified <see cref="IQueryable{T}"/> representing the paged data.</returns>
+    internal static IQueryable<T> ApplyODataPaging<T>(this IQueryable<T> query, ODataQueryOptions<T> options, ODataQuerySettings settings)
+    {
+        int takeValue = Math.Min(options.Top?.Value ?? int.MaxValue, settings.PageSize ?? 100);
+        int skipValue = Math.Max(options.Skip?.Value ?? 0, 0);
+        return query.Skip(skipValue).Take(takeValue);
+    }
+
+    /// <summary>
+    /// Applies the <c>$select</c> OData query option to the provided query.
+    /// </summary>
+    /// <typeparam name="T">The type of entity being queried.</typeparam>
+    /// <param name="dataset">The datset to apply the <c>$select</c> option to.</param>
+    /// <param name="queryOption">The <see cref="SelectExpandQueryOption"/> to apply.</param>
+    /// <param name="settings">The query settings being used.</param>
+    /// <returns>The resulting dataset after property selection.</returns>
+    internal static IEnumerable<object> ApplyODataSelect<T>(this IList<T> dataset, SelectExpandQueryOption? queryOption, ODataQuerySettings settings)
+    {
+        if (dataset.Count == 0 || queryOption is null)
+        {
+            return dataset.Cast<object>().AsEnumerable();
+        }
+
+        return queryOption.ApplyTo(dataset.AsQueryable(), settings).Cast<object>().ToList().Select(x => ((ISelectExpandWrapper)x).ToDictionary());
+    }
+
     /// <summary>
     /// Determines if the provided entity is in the view of the current user.
     /// </summary>