Enforces default Top for CDP Connector for non delegating expression. (#2904)

This pull request introduces a new delegation parameter class to handle
`$top` query parameters for paginated data retrieval, updates the
`ConnectorSettings` class to support configurable maximum row limits,
and integrates these changes into the existing `CdpTable` and
`CdpTableValue` implementations. It also includes updates to tests to
validate the new functionality.

### Enhancements for pagination and delegation:

* **Added `DefaultCDPDelegationParameter` class**: This new class
implements delegation parameters to generate OData query strings,
including `$top` for row limits.
(`src/libraries/Microsoft.PowerFx.Connectors/Tabular/DefaultCDPDelegationParameter.cs`)

* **Updated `CdpTableValue.Rows` property**: Modified to use the new
`DefaultCDPDelegationParameter` for setting the maximum number of rows
via `$top`.
(`src/libraries/Microsoft.PowerFx.Connectors/Public/CdpTableValue.cs`)

### Configuration improvements:

* **Enhanced `ConnectorSettings`**: Introduced a `DefaultConnectorTop`
constant and updated the `MaxRows` property to use this default value,
with the option to override it when creating new connector settings.
(`src/libraries/Microsoft.PowerFx.Connectors/Public/ConnectorSettings.cs`)
[[1]](diffhunk://#diff-5b668106d359b647dc2d970cc4c94e742319a83eea6389adb346c39ac208f4b3L14-R26)
[[2]](diffhunk://#diff-5b668106d359b647dc2d970cc4c94e742319a83eea6389adb346c39ac208f4b3L41-R47)

### Integration with `CdpTable`:

* **Exposed `ConnectorSettings` in `CdpTable`**: Added a new property to
provide access to `ConnectorSettings` from `CdpTable`.
(`src/libraries/Microsoft.PowerFx.Connectors/Tabular/Services/CdpTable.cs`)

* **Utilized delegation parameters in `Query` method**: Updated the
query logic to use `DefaultCDPDelegationParameter` for constructing
OData query strings with `$top`.
(`src/libraries/Microsoft.PowerFx.Connectors/Tabular/Services/CdpTable.cs`)

### Test updates:

* **Modified test cases**: Updated tests to validate the correct
generation of `$top` query parameters and ensure the new functionality
works as expected.
(`src/tests/Microsoft.PowerFx.Connectors.Tests.Shared/PowerPlatformTabularTests.cs`)
[[1]](diffhunk://#diff-7b1be2cff9606912fdb9b09ff75af4e880829323c9e42a7252a1f93feb1ded67L82-R82)
[[2]](diffhunk://#diff-7b1be2cff9606912fdb9b09ff75af4e880829323c9e42a7252a1f93feb1ded67R143-R146)

Author: jas-valgotar

src/libraries/Microsoft.PowerFx.Connectors/Public/CdpTableValue.cs

@@ -7,6 +7,7 @@
 using System.Net.Http;
 using System.Threading;
 using System.Threading.Tasks;
+using Microsoft.PowerFx.Connectors.Tabular;
 using Microsoft.PowerFx.Core.Entities;
 using Microsoft.PowerFx.Core.IR;
 using Microsoft.PowerFx.Types;
@@ -43,7 +44,7 @@ internal CdpTableValue(IRContext irContext)
             _cachedRows = null;
         }
 
-        public override IEnumerable<DValue<RecordValue>> Rows => GetRowsAsync(null, null, CancellationToken.None).ConfigureAwait(false).GetAwaiter().GetResult();
+        public override IEnumerable<DValue<RecordValue>> Rows => GetRowsAsync(null, new DefaultCDPDelegationParameter(RecordType.ToTable(), _tabularService.ConnectorSettings.MaxRows), CancellationToken.None).ConfigureAwait(false).GetAwaiter().GetResult();
 
         public DelegationParameterFeatures SupportedFeatures => DelegationParameterFeatures.Filter |
                 DelegationParameterFeatures.Top |

src/libraries/Microsoft.PowerFx.Connectors/Public/ConnectorSettings.cs

@@ -11,13 +11,19 @@ namespace Microsoft.PowerFx.Connectors
     [ThreadSafeImmutable]
     public class ConnectorSettings
     {
-        public static ConnectorSettings NewCDPConnectorSettings(bool extractSensitivityLabel = false, string purviewAccountName = null)
+        /// <summary>
+        /// Default number of rows to return for connector, per page. I.E. $top=1000.
+        /// </summary>
+        internal const int DefaultConnectorTop = 1000;
+
+        public static ConnectorSettings NewCDPConnectorSettings(bool extractSensitivityLabel = false, string purviewAccountName = null, int maxRows = DefaultConnectorTop)
         {
             var connectorSettings = new ConnectorSettings(null)
             {
                 Compatibility = ConnectorCompatibility.CdpCompatibility,
                 SupportXMsEnumValues = true,
                 ReturnEnumsAsPrimitive = false,
+                MaxRows = maxRows,
                 ExtractSensitivityLabel = extractSensitivityLabel,
                 PurviewAccountName = purviewAccountName
             };
@@ -38,7 +44,7 @@ public ConnectorSettings(string @namespace)
         /// <summary>
         /// Maximum number of rows to return, per page.
         /// </summary>
-        public int MaxRows { get; init; } = 1000;
+        public int MaxRows { get; init; } = DefaultConnectorTop;
 
         /// <summary>
         /// If this is enabled it will extract MIP Labels.

src/libraries/Microsoft.PowerFx.Connectors/Tabular/DefaultCDPDelegationParameter.cs

@@ -0,0 +1,54 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT license.
+
+using System;
+using System.Collections.Generic;
+using System.Text;
+using Microsoft.PowerFx.Types;
+
+namespace Microsoft.PowerFx.Connectors.Tabular
+{
+    internal class DefaultCDPDelegationParameter : DelegationParameters
+    {
+        private readonly int _maxRows;
+
+        public override DelegationParameterFeatures Features => throw new NotImplementedException();
+
+        private readonly FormulaType _expectedReturnType;
+
+        public override FormulaType ExpectedReturnType => throw new NotImplementedException();
+
+        public DefaultCDPDelegationParameter(FormulaType expectedReturnType, int maxRows)
+        {
+            _expectedReturnType = expectedReturnType;
+            _maxRows = maxRows;
+        }
+
+        public override string GetODataApply()
+        {
+            return string.Empty;
+        }
+
+        public override string GetOdataFilter()
+        {
+            return string.Empty;
+        }
+
+        public override string GetODataQueryString()
+        {
+            var sb = new StringBuilder();
+            sb.Append($"$top={_maxRows}");
+            return sb.ToString();
+        }
+
+        public override IReadOnlyCollection<(string, bool)> GetOrderBy()
+        {
+            return Array.Empty<(string, bool)>();
+        }
+
+        public override bool ReturnTotalCount()
+        {
+            return false;
+        }
+    }
+}

src/libraries/Microsoft.PowerFx.Connectors/Tabular/Services/CdpService.cs

@@ -26,6 +26,8 @@ public abstract class CdpService : CdpServiceBase
 
         public abstract HttpClient HttpClient { get; }
 
+        public abstract ConnectorSettings ConnectorSettings { get; }
+
         public virtual CdpTableValue GetTableValue()
         {
             return IsInitialized

src/libraries/Microsoft.PowerFx.Connectors/Tabular/Services/CdpTable.cs

@@ -8,6 +8,7 @@
 using System.Net.Http;
 using System.Threading;
 using System.Threading.Tasks;
+using Microsoft.PowerFx.Connectors.Tabular;
 using Microsoft.PowerFx.Core.Entities;
 using Microsoft.PowerFx.Core.IR;
 using Microsoft.PowerFx.Types;
@@ -53,6 +54,8 @@ public class CdpTable : CdpService
 
         private readonly ConnectorSettings _connectorSettings;
 
+        public override ConnectorSettings ConnectorSettings => _connectorSettings;
+
         internal CdpTable(string dataset, string table, IReadOnlyCollection<RawTable> tables, ConnectorSettings connectorSettings,  CDPMetadataItem fieldMetadata = null)
         {
             DatasetName = dataset ?? throw new ArgumentNullException(nameof(dataset));
@@ -138,7 +141,8 @@ private async Task<string> Query(IServiceProvider serviceProvider, DelegationPar
         {
             cancellationToken.ThrowIfCancellationRequested();
             ConnectorLogger executionLogger = serviceProvider?.GetService<ConnectorLogger>();
-            string queryParams = parameters?.GetODataQueryString() ?? string.Empty;
+            parameters ??= new DefaultCDPDelegationParameter(ConnnectorType.FormulaType, _connectorSettings.MaxRows);
+            string queryParams = parameters.GetODataQueryString();
             if (!string.IsNullOrEmpty(queryParams))
             {
                 queryParams = "&" + queryParams;

src/tests/Microsoft.PowerFx.Connectors.Tests.Shared/FileTabularConnector.cs

@@ -82,6 +82,8 @@ public FileTabularService(string fileName)
         // No need for files
         public override HttpClient HttpClient => null;
 
+        public override ConnectorSettings ConnectorSettings => ConnectorSettings.NewCDPConnectorSettings();
+
         internal override IReadOnlyDictionary<string, Relationship> Relationships => null;
 
         // Initialization can be synchronous

src/tests/Microsoft.PowerFx.Connectors.Tests.Shared/PowerPlatformTabularTests.cs

@@ -79,7 +79,7 @@ public async Task SQL_CdpTabular_GetTables()
             Assert.Equal("DisplayName", dm.Parameters.Skip(1).First().XMsDynamicValues.ValueTitle);
             Assert.Equal("Database name", dm.Parameters.Skip(1).First().XMsSummary);
 
-            CdpDataSource cds = new CdpDataSource("pfxdev-sql.database.windows.net,connectortest");
+            CdpDataSource cds = new CdpDataSource("pfxdev-sql.database.windows.net,connectortest", ConnectorSettings.NewCDPConnectorSettings(maxRows: 101));
 
             testConnector.SetResponseFromFiles(@"Responses\SQL GetDatasetsMetadata.json", @"Responses\SQL GetTables.json");
             IEnumerable<CdpTable> tables = await cds.GetTablesAsync(client, $"/apim/sql/{connectionId}", CancellationToken.None, logger);
@@ -140,6 +140,10 @@ public async Task SQL_CdpTabular_GetTables()
             // Rows are not cached here as the cache is stored in CdpTableValue which is created by InjectServiceProviderFunction, itself added during Engine.Check
             testConnector.SetResponseFromFile(@"Responses\SQL Server Get First Customers.json");
             result = await engine.EvalAsync("Last(Customers).Phone", CancellationToken.None, runtimeConfig: rc);
+
+            // Verify that we have the right URL with correct $top
+            Assert.Contains("x-ms-request-url: /apim/sql/c1a4e9f52ec94d55bb82f319b3e33a6a/v2/datasets/pfxdev-sql.database.windows.net%2Cconnectortest/tables/%5Bdbo%5D.%5BCustomers%5D/items?api-version=2015-09-01&$top=101", testConnector._log.ToString(), StringComparison.InvariantCulture);
+
             StringValue phone = Assert.IsType<StringValue>(result);
             Assert.Equal("+1-425-705-0000", phone.Value);
         }