tests(Spanner): Fix flake in transaction scope tests.

Two transaction scope tests modify the same row. Since tests run in parallel, on ocassion the transaction for one is aborted. The retry mechanism we had only looks at Spanner exceptions, but since these tests are testing System.Transactions integration, the exception we get is System.Transactions.TransactionAbortedException. This commit modifies the existing retry mechanism to accept custom retry conditions so that we can retry on these cases.

Author: amanda-tarafa

apis/Google.Cloud.Spanner.Data/Google.Cloud.Spanner.Data.CommonTesting/RetryHelpers.cs

@@ -47,19 +47,21 @@ public static Task<int> ExecuteNonQueryAsyncWithRetry(this SpannerCommand comman
         /// <summary>
         /// Executes the given action, retrying once if the first attempt is aborted.
         /// </summary>
-        public static void ExecuteWithRetry(Action action) => ExecuteWithRetryImpl(() => { action(); return 0; });
+        public static void ExecuteWithRetry(Action action, Func<Exception, bool> shouldRetry = null) => 
+            ExecuteWithRetryImpl(() => { action(); return 0; }, shouldRetry);
 
         /// <summary>
         /// Executes the given asynchronous action, retrying once if the first attempt is aborted.
         /// </summary>
-        public static Task ExecuteWithRetryAsync(Func<Task> action) => ExecuteWithRetryAsyncImpl(async () => { await action(); return 0; });
+        public static Task ExecuteWithRetryAsync(Func<Task> action, Func<Exception, bool> shouldRetry = null) => 
+            ExecuteWithRetryAsyncImpl(async () => { await action(); return 0; }, shouldRetry);
 
         private static int _calls;
         private static int _retries;
 
         // TODO: Move this retry code into production code, so that everyone can use it.
 
-        private static T ExecuteWithRetryImpl<T>(Func<T> func)
+        private static T ExecuteWithRetryImpl<T>(Func<T> func, Func<Exception, bool> shouldRetry = null)
         {
             Interlocked.Increment(ref _calls);
 
@@ -76,15 +78,15 @@ private static T ExecuteWithRetryImpl<T>(Func<T> func)
                 {
                     return func();
                 }
-                catch (SpannerException e) when (attempt.ShouldRetry(e))
+                catch (Exception e) when (attempt.ShouldRetry(e) || (shouldRetry?.Invoke(e) ?? false))
                 {
                     attempt.Backoff(default);
                 }
             }
             throw new InvalidOperationException("Bug in GAX retry handling: finished sequence of attempts");
         }
 
-        private static async Task<T> ExecuteWithRetryAsyncImpl<T>(Func<Task<T>> func)
+        private static async Task<T> ExecuteWithRetryAsyncImpl<T>(Func<Task<T>> func, Func<Exception, bool> shouldRetry = null)
         {
             Interlocked.Increment(ref _calls);
 
@@ -101,7 +103,7 @@ private static async Task<T> ExecuteWithRetryAsyncImpl<T>(Func<Task<T>> func)
                 {
                     return await func();
                 }
-                catch (SpannerException e) when (attempt.ShouldRetry(e))
+                catch (Exception e) when (attempt.ShouldRetry(e) || (shouldRetry?.Invoke(e) ?? false))
                 {
                     await attempt.BackoffAsync(default);
                 }

apis/Google.Cloud.Spanner.Data/Google.Cloud.Spanner.Data.IntegrationTests/TransactionScopeTests.cs

@@ -209,7 +209,7 @@ await RetryHelpers.ExecuteWithRetryAsync(async () =>
                     await UpdateValueAsync(writeConnection);
                     scope.Complete();
                 }
-            });
+            }, ex => ex is TransactionAbortedException);
         }
 
         [Fact]
@@ -224,7 +224,7 @@ public void OneWrite_Success()
                     UpdateValue(writeConnection);
                     scope.Complete();
                 }
-            });
+            }, ex => ex is TransactionAbortedException);
         }
 
         [Fact]