extending retries for scale metric table creation (#5611)

Author: brettsam

src/WebJobs.Script.WebHost/Scale/TableStorageScaleMetricsRepository.cs

@@ -25,19 +25,27 @@ public class TableStorageScaleMetricsRepository : IScaleMetricsRepository
         private const string MonitorIdPropertyName = "MonitorId";
         private const string SampleTimestampPropertyName = "SampleTimestamp";
         private const int MetricsPurgeDelaySeconds = 30;
+        private const int DefaultTableCreationRetries = 3;
 
         private readonly IConfiguration _configuration;
         private readonly IHostIdProvider _hostIdProvider;
         private readonly ScaleOptions _scaleOptions;
         private readonly ILogger _logger;
+        private readonly int _tableCreationRetries;
         private CloudTableClient _tableClient;
 
         public TableStorageScaleMetricsRepository(IConfiguration configuration, IHostIdProvider hostIdProvider, IOptions<ScaleOptions> scaleOptions, ILoggerFactory loggerFactory)
+            : this(configuration, hostIdProvider, scaleOptions, loggerFactory, DefaultTableCreationRetries)
+        {
+        }
+
+        internal TableStorageScaleMetricsRepository(IConfiguration configuration, IHostIdProvider hostIdProvider, IOptions<ScaleOptions> scaleOptions, ILoggerFactory loggerFactory, int tableCreationRetries)
         {
             _configuration = configuration;
             _hostIdProvider = hostIdProvider;
             _scaleOptions = scaleOptions.Value;
             _logger = loggerFactory.CreateLogger<TableStorageScaleMetricsRepository>();
+            _tableCreationRetries = tableCreationRetries;
         }
 
         internal CloudTableClient TableClient
@@ -199,7 +207,7 @@ internal async Task ExecuteBatchSafeAsync(TableBatchOperation batch, DateTime? n
             }
         }
 
-        internal async Task CreateIfNotExistsAsync(CloudTable table, int retryCount = 3, int retryDelayMS = 1000)
+        internal async Task CreateIfNotExistsAsync(CloudTable table, int retryDelayMS = 1000)
         {
             int attempt = 0;
             do
@@ -222,7 +230,8 @@ internal async Task CreateIfNotExistsAsync(CloudTable table, int retryCount = 3,
                     // though these should only happen in tests not production, because we only ever
                     // delete OLD tables and we'll never be attempting to recreate a table we just
                     // deleted outside of tests.
-                    if (e.RequestInformation.HttpStatusCode == (int)HttpStatusCode.Conflict)
+                    if (e.RequestInformation.HttpStatusCode == (int)HttpStatusCode.Conflict &&
+                        attempt < _tableCreationRetries)
                     {
                         // wait a bit and try again
                         await Task.Delay(retryDelayMS);
@@ -233,7 +242,7 @@ internal async Task CreateIfNotExistsAsync(CloudTable table, int retryCount = 3,
 
                 return;
             }
-            while (attempt++ < retryCount);
+            while (attempt++ < _tableCreationRetries);
         }
 
         internal async Task AccumulateMetricsBatchAsync(TableBatchOperation batch, IScaleMonitor monitor, IEnumerable<ScaleMetrics> metrics, DateTime? now = null)

test/WebJobs.Script.Tests.Integration/Scale/TableStorageScaleMetricsRepositoryTests.cs

@@ -41,7 +41,9 @@ public TableStorageScaleMetricsRepositoryTests()
             _loggerProvider = new TestLoggerProvider();
             ILoggerFactory loggerFactory = new LoggerFactory();
             loggerFactory.AddProvider(_loggerProvider);
-            _repository = new TableStorageScaleMetricsRepository(configuration, _hostIdProviderMock.Object, new OptionsWrapper<ScaleOptions>(_scaleOptions), loggerFactory);
+
+            // Allow for up to 30 seconds of creation retries for tests due to slow table deletes
+            _repository = new TableStorageScaleMetricsRepository(configuration, _hostIdProviderMock.Object, new OptionsWrapper<ScaleOptions>(_scaleOptions), loggerFactory, 30);
 
             EmptyMetricsTableAsync().GetAwaiter().GetResult();
         }
@@ -163,7 +165,7 @@ public async Task ReadMetricsAsync_FiltersExpiredMetrics()
             }
 
             await _repository.ExecuteBatchSafeAsync(batch);
-            
+
             var result = await _repository.ReadMetricsAsync(monitors);
 
             var resultMetrics = result[monitor1].Cast<TestScaleMetrics1>().ToArray();
@@ -327,7 +329,7 @@ await TestHelpers.Await(async () =>
         public async Task LogStorageException_LogsDetails()
         {
             StorageException ex = null;
-            var table =_repository.TableClient.GetTableReference("dne");
+            var table = _repository.TableClient.GetTableReference("dne");
             var continuationToken = new TableContinuationToken();
             try
             {