updating v11 retry mechanism

Author: Derek Legenzoff

optimize-data-indexing/v11/OptimizeDataIndexing/ExponentialBackoff.cs

@@ -34,22 +34,22 @@ private static async Task<IndexDocumentsResult> ExponentialBackoffAsync(SearchCl
 
                     var failedDocuments = result.Results.Where(r => r.Succeeded != true).ToList();
 
+                    // handle partial failure
                     if (failedDocuments.Count > 0)
                     {
-                        Console.WriteLine("BATCH STARTING AT DOC {0}:", id);
-                        Console.WriteLine("[Attempt: {0} of {1} Failed] - Error: {2} \n", attempts, maxRetryAttempts);
-
+                        
                         if (attempts == maxRetryAttempts)
                         {
-                            Console.WriteLine("BATCH STARTING AT DOC {0}:", id);
                             Console.WriteLine("[MAX RETRIES HIT] - Giving up on the batch starting at {0}", id);
                             break;
                         } 
                         else
                         {
-                            Console.WriteLine("BATCH STARTING AT DOC {0}:", id);
-                            Console.WriteLine("[Attempt: {0} of {1} Failed] - Error: {2} \n", attempts, maxRetryAttempts);
-                            Console.WriteLine("{0} documents failed:", failedDocuments.Count);
+                            Console.WriteLine("[Batch starting at doc {0} had partial failure]", id);
+                            //Console.WriteLine("[Attempt: {0} of {1} Failed]", attempts, maxRetryAttempts);
+                            Console.WriteLine("[Retrying {0} failed documents] \n", failedDocuments.Count);
+
+                            // creating a batch of failed documents to retry
                             var failedDocumentKeys = failedDocuments.Select(doc => doc.Key).ToList();
                             hotels = hotels.Where(h => failedDocumentKeys.Contains(h.HotelId)).ToList();
                             batch = IndexDocumentsBatch.Upload(hotels);
@@ -65,8 +65,9 @@ private static async Task<IndexDocumentsResult> ExponentialBackoffAsync(SearchCl
                 }
                 catch (RequestFailedException ex)
                 {
-                    Console.WriteLine("BATCH STARTING AT DOC {0}:", id);
-                    Console.WriteLine("[Attempt: {0} of {1} Failed] - Error: {2} \n", attempts, maxRetryAttempts, ex.Message);
+                    Console.WriteLine("[Batch starting at doc {0} failed]", id);
+                    //Console.WriteLine("[Attempt: {0} of {1} Failed] - Error: {2} \n", attempts, maxRetryAttempts, ex.Message);
+                    Console.WriteLine("[Retrying entire batch] \n");
 
                     if (attempts == maxRetryAttempts)
                     {

optimize-data-indexing/v11/OptimizeDataIndexing/Program.cs

@@ -35,18 +35,18 @@ public static async Task Main(string[] args)
             Console.WriteLine("{0}", "Creating index...\n");
             await CreateIndexAsync(indexName, indexClient);
 
-            //Console.WriteLine("{0}", "Finding optimal batch size...\n");
-            //await TestBatchSizesAsync(searchClient, numTries: 3);
+            Console.WriteLine("{0}", "Finding optimal batch size...\n");
+            await TestBatchSizesAsync(searchClient, numTries: 3);
 
-            long numDocuments = 100000;
-            DataGenerator dg = new DataGenerator();
-            List<Hotel> hotels = dg.GetHotels(numDocuments, "large");
+            //long numDocuments = 100000;
+            //DataGenerator dg = new DataGenerator();
+            //List<Hotel> hotels = dg.GetHotels(numDocuments, "large");
 
-            Console.WriteLine("{0}", "Uploading using exponential backoff...\n");
-            await ExponentialBackoff.IndexDataAsync(searchClient, hotels, 500, 20); //8
+            //Console.WriteLine("{0}", "Uploading using exponential backoff...\n");
+            //await ExponentialBackoff.IndexDataAsync(searchClient, hotels, 1000, 8);
 
-            Console.WriteLine("{0}", "Validating all data was indexed...\n");
-            await ValidateIndexAsync(indexClient, indexName, numDocuments);
+            //Console.WriteLine("{0}", "Validating all data was indexed...\n");
+            //await ValidateIndexAsync(indexClient, indexName, numDocuments);
 
             Console.WriteLine("{0}", "Complete.  Press any key to end application...\n");
             Console.ReadKey();