Added a duration cap (15s) to getParentInstanceId() methods (#267)

Author: scale-tone

durablefunctionsmonitor.dotnetbackend/Common/DetailedOrchestrationStatus.cs

@@ -14,6 +14,7 @@
 using System.Text.RegularExpressions;
 using Microsoft.Extensions.Logging;
 using Microsoft.WindowsAzure.Storage.Table;
+using System.Threading;
 
 namespace DurableFunctionsMonitor.DotNetBackend
 {
@@ -81,6 +82,8 @@ internal static async Task<DetailedOrchestrationStatus> CreateFrom(DurableOrches
             return result;
         }
 
+        private static int GetParentInstanceIdTimeoutInSeconds = 15;
+
         internal static async Task<string> GetParentInstanceIdDirectlyFromTable(IDurableClient durableClient, string connEnvVariableName, string hubName, string instanceId)
         {
             var tableClient = await TableClient.GetTableClient(connEnvVariableName);
@@ -131,7 +134,9 @@ internal static async Task<string> GetParentInstanceIdDirectlyFromTable(IDurable
                     )
                 );
 
-                tableResult = await tableClient.GetAllAsync($"{durableClient.TaskHubName}History", executionIdQuery);
+                // This scan can still take long time, so we'll have to hard-limit it to a few seconds. TaskCancelledException will be handled by upper code.
+                var cts = new CancellationTokenSource(TimeSpan.FromSeconds(GetParentInstanceIdTimeoutInSeconds));
+                tableResult = await tableClient.GetAllAsync($"{durableClient.TaskHubName}History", executionIdQuery, cts.Token);
             }
 
             return tableResult.FirstOrDefault()?.PartitionKey;

durablefunctionsmonitor.dotnetbackend/Common/TableClient.cs

@@ -8,6 +8,7 @@
 using Microsoft.WindowsAzure.Storage.Table;
 using Microsoft.WindowsAzure.Storage;
 using Microsoft.WindowsAzure.Storage.Auth;
+using System.Threading;
 
 namespace DurableFunctionsMonitor.DotNetBackend
 {
@@ -22,7 +23,10 @@ public interface ITableClient
 
         // Asynchronously retrieves all results from Azure Table
         Task<IEnumerable<TEntity>> GetAllAsync<TEntity>(string tableName, TableQuery<TEntity> query) where TEntity : TableEntity, new();
-        
+
+        // Asynchronously retrieves all results from Azure Table
+        Task<IEnumerable<TEntity>> GetAllAsync<TEntity>(string tableName, TableQuery<TEntity> query, CancellationToken ct) where TEntity : TableEntity, new();
+
         // Executes a TableOperation
         Task<TableResult> ExecuteAsync(string tableName, TableOperation operation);
     }
@@ -119,7 +123,11 @@ public IEnumerable<TEntity> GetAll<TEntity>(string tableName, TableQuery<TEntity
         }
 
         /// <inheritdoc/>
-        public async Task<IEnumerable<TEntity>> GetAllAsync<TEntity>(string tableName, TableQuery<TEntity> query)
+        public Task<IEnumerable<TEntity>> GetAllAsync<TEntity>(string tableName, TableQuery<TEntity> query)
+            where TEntity : TableEntity, new() => this.GetAllAsync(tableName, query, CancellationToken.None);
+
+        /// <inheritdoc/>
+        public async Task<IEnumerable<TEntity>> GetAllAsync<TEntity>(string tableName, TableQuery<TEntity> query, CancellationToken ct)
             where TEntity : TableEntity, new()
         {
             var table = this._client.GetTableReference(tableName);
@@ -134,7 +142,9 @@ public async Task<IEnumerable<TEntity>> GetAllAsync<TEntity>(string tableName, T
             TableContinuationToken token = null;
             do
             {
-                var nextBatch = await table.ExecuteQuerySegmentedAsync(query, token, null, operationContext);
+                ct.ThrowIfCancellationRequested();
+
+                var nextBatch = await table.ExecuteQuerySegmentedAsync(query, token, null, operationContext, ct);
 
                 result.AddRange(nextBatch.Results);
                 token = nextBatch.ContinuationToken;

durablefunctionsmonitor.dotnetisolated.core/Common/DetailedOrchestrationStatus.cs

@@ -81,6 +81,8 @@ internal static async Task<DetailedOrchestrationStatus> CreateFrom(
             return result;
         }
 
+        private static int GetParentInstanceIdTimeoutInSeconds = 15;
+
         internal static async Task<string> GetParentInstanceIdDirectlyFromTable(DurableTaskClient durableClient, string connEnvVariableName, string hubName, string instanceId)
         {
             var tableClient = await TableClient.GetTableClient(connEnvVariableName);
@@ -131,7 +133,9 @@ internal static async Task<string> GetParentInstanceIdDirectlyFromTable(DurableT
                     )
                 );
 
-                tableResult = await tableClient.GetAllAsync($"{durableClient.Name}History", executionIdQuery);
+                // This scan can still take long time, so we'll have to hard-limit it to a few seconds. TaskCancelledException will be handled by upper code.
+                var cts = new CancellationTokenSource(TimeSpan.FromSeconds(GetParentInstanceIdTimeoutInSeconds));
+                tableResult = await tableClient.GetAllAsync($"{durableClient.Name}History", executionIdQuery, cts.Token);
             }
 
             return tableResult.FirstOrDefault()?.PartitionKey;

durablefunctionsmonitor.dotnetisolated.core/Common/TableClient.cs

@@ -19,6 +19,9 @@ public interface ITableClient
         // Asynchronously retrieves all results from Azure Table
         Task<IEnumerable<TEntity>> GetAllAsync<TEntity>(string tableName, TableQuery<TEntity> query) where TEntity : TableEntity, new();
 
+        // Asynchronously retrieves all results from Azure Table
+        Task<IEnumerable<TEntity>> GetAllAsync<TEntity>(string tableName, TableQuery<TEntity> query, CancellationToken ct) where TEntity : TableEntity, new();
+
         // Executes a TableOperation
         Task<TableResult> ExecuteAsync(string tableName, TableOperation operation);
     }
@@ -122,7 +125,11 @@ public IEnumerable<TEntity> GetAll<TEntity>(string tableName, TableQuery<TEntity
         }
 
         /// <inheritdoc/>
-        public async Task<IEnumerable<TEntity>> GetAllAsync<TEntity>(string tableName, TableQuery<TEntity> query)
+        public Task<IEnumerable<TEntity>> GetAllAsync<TEntity>(string tableName, TableQuery<TEntity> query)
+            where TEntity : TableEntity, new() => this.GetAllAsync(tableName, query, CancellationToken.None);
+
+        /// <inheritdoc/>
+        public async Task<IEnumerable<TEntity>> GetAllAsync<TEntity>(string tableName, TableQuery<TEntity> query, CancellationToken ct)
             where TEntity : TableEntity, new()
         {
             var table = this._client.GetTableReference(tableName);
@@ -137,7 +144,9 @@ public async Task<IEnumerable<TEntity>> GetAllAsync<TEntity>(string tableName, T
             TableContinuationToken token = null;
             do
             {
-                var nextBatch = await table.ExecuteQuerySegmentedAsync(query, token, null, operationContext);
+                ct.ThrowIfCancellationRequested();
+
+                var nextBatch = await table.ExecuteQuerySegmentedAsync(query, token, null, operationContext, ct);
 
                 result.AddRange(nextBatch.Results);
                 token = nextBatch.ContinuationToken;