Address part of race condition #149: add git retry headroom

Add some headroom among git clone retries (5s=60s:CC default polling period -55s: retry clone duration)

Author: gberche-orange

cf-ops-automation-broker-core/src/main/java/com/orange/oss/cloudfoundry/broker/opsautomation/ondemandbroker/git/RetrierGitManager.java

@@ -18,10 +18,13 @@ public RetrierGitManager(String repositoryAliasName, GitManager gitManager, Retr
         this.retryPolicy = retryPolicy;
         this.retryPolicy
                 .onRetry(e -> logger.warn("Transient (?) failure, retrying. Cause: {}",  e.getLastFailure()))
-                .onRetriesExceeded(e -> logger.warn("Aborting. Max attempts reached: #" + this.retryPolicy.getMaxAttempts() + " Rethrowing failure:" + e.getFailure()))
+                .onRetriesExceeded(e -> logger.warn("Aborting. Max attempts reached: #" + this.retryPolicy.getMaxAttempts() +
+                        " or max duration reached (" + this.retryPolicy.getMaxDuration().toString() + "). Rethrowing failure:" + e.getFailure()))
                 .handleIf(e -> isCauseSubclassOf(e, org.eclipse.jgit.api.errors.TransportException.class))
         ;
         logger.debug("Configured for {} with retry policy {}", repositoryAliasName, ToStringBuilder.reflectionToString(retryPolicy));
+        //Duration would typically be displayed as "PT2S" which means P (for duration prefixes), T (for time in units smaller than the day, in our case, usually seconds or minutes
+        //Learn more at https://en.wikipedia.org/wiki/ISO_8601#Durations
     }
 
     protected boolean isCauseSubclassOf(Throwable e, Class superClassToCheck) {

cf-ops-automation-broker-core/src/main/java/com/orange/oss/cloudfoundry/broker/opsautomation/ondemandbroker/git/RetryProperties.java

@@ -23,7 +23,7 @@ public class RetryProperties {
      * the max duration to perform retries for, else the execution will be failed.
      * See {@link RetryPolicy#withMaxDuration(Duration)}
      */
-    private int maxDurationMilliSeconds=60000;
+    private int maxDurationMilliSeconds=50000;
 
     public int getMaxAttempts() { return maxAttempts; }
     public void setMaxAttempts(int maxAttempts) { this.maxAttempts = maxAttempts; }

cf-ops-automation-broker-core/src/test/java/com/orange/oss/cloudfoundry/broker/opsautomation/ondemandbroker/git/RetrierGitManagerTest.java

@@ -7,8 +7,12 @@
 import org.junit.Test;
 import org.junit.runner.RunWith;
 import org.mockito.Mock;
+import org.mockito.internal.stubbing.answers.AnswersWithDelay;
+import org.mockito.internal.stubbing.answers.ThrowsException;
 import org.mockito.junit.MockitoJUnitRunner;
 
+import java.time.Duration;
+
 import static org.mockito.ArgumentMatchers.any;
 import static org.mockito.Mockito.*;
 
@@ -20,6 +24,35 @@ public class RetrierGitManagerTest {
 
 
 
+    @Test
+    public void retries_clones_until_max_duration_reached() {
+        //Given a retrier is configured with a retry policy and a max retry duration of 2s
+        RetryPolicy<Object> retryPolicy = new RetryPolicy<>()
+                .withMaxAttempts(3)
+                .withMaxDuration(Duration.ofMillis(2*1000));
+        GitManager retrier = new RetrierGitManager("repoAlias", gitManager, retryPolicy);
+
+        //Given 2 network problems when trying to clone
+        TransportException gitException = new TransportException("https://elpaaso-gitlab.mycompany.com/paas-templates.git: 502 Bad Gateway");
+        IllegalArgumentException wrappedException = new IllegalArgumentException(gitException);
+        //Inject delay, see https://stackoverflow.com/questions/12813881/can-i-delay-a-stubbed-method-response-with-mockito
+        doAnswer( new AnswersWithDelay( 3*1000,  new ThrowsException(wrappedException))). //1st attempt consumming max retry time budget
+                doNothing(). //2nd attempt that should not happen
+                doNothing(). //3nd attempt that should not happen
+                when(gitManager).cloneRepo(any());
+
+        try {
+            //when trying to clone
+            retrier.cloneRepo(new Context());
+            //then it rethrows the exception
+            Assertions.fail("expected max attempts reached to rethrow last exception, as max duration exceeded");
+        } catch (IllegalArgumentException e) {
+            verify(gitManager, times(1)).cloneRepo(any());
+            //success
+        }
+
+    }
+
     @Test
     public void retries_clones() {
         //Given a retrier is configured with a retry policy