Merge pull request #920 from simonejsing/simon/concurrentcachewrite

Gracefully handle IOException on concurrent cache writes

Author: JakeGinnivan

src/GitVersionCore.Tests/GitVersionCore.Tests.csproj

@@ -113,6 +113,8 @@
     <Compile Include="ConfigProviderTests.cs" />
     <Compile Include="GitVersionContextTests.cs" />
     <Compile Include="Helpers\DirectoryHelper.cs" />
+    <Compile Include="Mocks\MockThreadSleep.cs" />
+    <Compile Include="OperationWithExponentialBackoffTests.cs" />
     <Compile Include="Init\InitScenarios.cs" />
     <Compile Include="Init\InitStepsDefaultResponsesDoNotThrow.cs" />
     <Compile Include="Init\TestConsole.cs" />

src/GitVersionCore.Tests/Mocks/MockThreadSleep.cs

@@ -0,0 +1,20 @@
+using System;
+using GitVersion.Helpers;
+
+public class MockThreadSleep : IThreadSleep
+{
+    private Action<int> Validator;
+     
+    public MockThreadSleep(Action<int> validator = null)
+    {
+        this.Validator = validator;
+    }
+
+    public void Sleep(int milliseconds)
+    {
+        if (Validator != null)
+        {
+            Validator(milliseconds);
+        }
+    }
+}

src/GitVersionCore.Tests/OperationWithExponentialBackoffTests.cs

@@ -0,0 +1,125 @@
+using System;
+using System.IO;
+using GitVersion.Helpers;
+using NUnit.Framework;
+using Shouldly;
+
+[TestFixture]
+public class OperationWithExponentialBackoffTests
+{
+    [Test]
+    public void RetryOperationThrowsWhenNegativeMaxRetries()
+    {
+        Action action = () => new OperationWithExponentialBackoff<IOException>(new MockThreadSleep(), () => { }, -1);
+        action.ShouldThrow<ArgumentOutOfRangeException>();
+    }
+
+    [Test]
+    public void RetryOperationThrowsWhenThreadSleepIsNull()
+    {
+        Action action = () => new OperationWithExponentialBackoff<IOException>(null, () => { });
+        action.ShouldThrow<ArgumentNullException>();
+    }
+
+    [Test]
+    public void OperationIsNotRetriedOnInvalidException()
+    {
+        Action operation = () =>
+        {
+            throw new Exception();
+        };
+
+        var retryOperation = new OperationWithExponentialBackoff<IOException>(new MockThreadSleep(), operation);
+        Action action = () => retryOperation.Execute();
+        action.ShouldThrow<Exception>();
+    }
+
+    [Test]
+    public void OperationIsRetriedOnIOException()
+    {
+        var operationCount = 0;
+
+        Action operation = () =>
+        {
+            operationCount++;
+            if (operationCount < 2)
+            {
+                throw new IOException();
+            }
+        };
+
+        var retryOperation = new OperationWithExponentialBackoff<IOException>(new MockThreadSleep(), operation);
+        retryOperation.Execute();
+
+        operationCount.ShouldBe(2);
+    }
+
+    [Test]
+    public void OperationIsRetriedAMaximumNumberOfTimes()
+    {
+        const int numberOfRetries = 3;
+        var operationCount = 0;
+
+        Action operation = () =>
+        {
+            operationCount++;
+            throw new IOException();
+        };
+
+        var retryOperation = new OperationWithExponentialBackoff<IOException>(new MockThreadSleep(), operation, numberOfRetries);
+        Action action = () => retryOperation.Execute();
+        action.ShouldThrow<AggregateException>();
+
+        operationCount.ShouldBe(numberOfRetries + 1);
+    }
+
+    [Test]
+    public void OperationDelayDoublesBetweenRetries()
+    {
+        const int numberOfRetries = 3;
+        var expectedSleepMSec = 500;
+        var sleepCount = 0;
+
+        Action operation = () =>
+        {
+            throw new IOException();
+        };
+
+        Action<int> validator = u =>
+        {
+            sleepCount++;
+            u.ShouldBe(expectedSleepMSec);
+            expectedSleepMSec *= 2;
+        };
+
+        var retryOperation = new OperationWithExponentialBackoff<IOException>(new MockThreadSleep(validator), operation, numberOfRetries);
+        Action action = () => retryOperation.Execute();
+        action.ShouldThrow<AggregateException>();
+
+        sleepCount.ShouldBe(numberOfRetries);
+    }
+
+    [Test]
+    public void TotalSleepTimeForSixRetriesIsAboutThirtySeconds()
+    {
+        const int numberOfRetries = 6;
+        int totalSleep = 0;
+
+        Action operation = () =>
+        {
+            throw new IOException();
+        };
+
+        Action<int> validator = u =>
+        {
+            totalSleep += u;
+        };
+
+        var retryOperation = new OperationWithExponentialBackoff<IOException>(new MockThreadSleep(validator), operation, numberOfRetries);
+        Action action = () => retryOperation.Execute();
+        action.ShouldThrow<AggregateException>();
+
+        // Exact number is 31,5 seconds
+        totalSleep.ShouldBe(31500);
+    }
+}

src/GitVersionCore/ExecuteCore.cs

@@ -2,6 +2,7 @@ namespace GitVersion
 {
     using System;
     using System.ComponentModel;
+    using System.IO;
     using System.Linq;
     using GitVersion.Helpers;
 
@@ -57,7 +58,14 @@ public VersionVariables ExecuteGitVersion(string targetUrl, string dynamicReposi
                 if (versionVariables == null)
                 {
                     versionVariables = ExecuteInternal(targetBranch, commitId, repo, gitPreparer, projectRoot, buildServer, overrideConfig: overrideConfig);
-                    gitVersionCache.WriteVariablesToDiskCache(repo, dotGitDirectory, versionVariables);
+                    try
+                    {
+                        gitVersionCache.WriteVariablesToDiskCache(repo, dotGitDirectory, versionVariables);
+                    }
+                    catch (AggregateException e)
+                    {
+                        Logger.WriteWarning(string.Format("One or more exceptions during cache write:{0}{1}", Environment.NewLine, e));
+                    }
                 }
 
                 return versionVariables;

src/GitVersionCore/GitVersionCache.cs

@@ -24,23 +24,29 @@ public void WriteVariablesToDiskCache(IRepository repo, string gitDir, VersionVa
             var cacheFileName = GetCacheFileName(GetKey(repo, gitDir), GetCacheDir(gitDir));
             variablesFromCache.FileName = cacheFileName;
 
-            using (var stream = fileSystem.OpenWrite(cacheFileName))
+            Dictionary<string, string> dictionary;
+            using (Logger.IndentLog("Creating dictionary"))
             {
-                using (var sw = new StreamWriter(stream))
-                {
-                    Dictionary<string, string> dictionary;
-                    using (Logger.IndentLog("Creating dictionary"))
-                    {
-                        dictionary = variablesFromCache.ToDictionary(x => x.Key, x => x.Value);
-                    }
+                dictionary = variablesFromCache.ToDictionary(x => x.Key, x => x.Value);
+            }
 
-                    using (Logger.IndentLog("Storing version variables to cache file " + cacheFileName))
+            Action writeCacheOperation = () =>
+            {
+                using (var stream = fileSystem.OpenWrite(cacheFileName))
+                {
+                    using (var sw = new StreamWriter(stream))
                     {
-                        var serializer = new Serializer();
-                        serializer.Serialize(sw, dictionary);
+                        using (Logger.IndentLog("Storing version variables to cache file " + cacheFileName))
+                        {
+                            var serializer = new Serializer();
+                            serializer.Serialize(sw, dictionary);
+                        }
                     }
                 }
-            }
+            };
+
+            var retryOperation = new OperationWithExponentialBackoff<IOException>(new ThreadSleep(), writeCacheOperation, maxRetries: 6);
+            retryOperation.Execute();
         }
 
         public VersionVariables LoadVersionVariablesFromDiskCache(IRepository repo, string gitDir)

src/GitVersionCore/GitVersionCore.csproj

@@ -122,7 +122,10 @@
     <Compile Include="GitVersionCache.cs" />
     <Compile Include="Helpers\FileSystem.cs" />
     <Compile Include="Helpers\IFileSystem.cs" />
+    <Compile Include="Helpers\IThreadSleep.cs" />
+    <Compile Include="Helpers\OperationWithExponentialBackoff.cs" />
     <Compile Include="Helpers\ServiceMessageEscapeHelper.cs" />
+    <Compile Include="Helpers\ThreadSleep.cs" />
     <Compile Include="IncrementStrategyFinder.cs" />
     <Compile Include="OutputVariables\VersionVariables.cs" />
     <Compile Include="SemanticVersionExtensions.cs" />

src/GitVersionCore/Helpers/IThreadSleep.cs

@@ -0,0 +1,7 @@
+namespace GitVersion.Helpers
+{
+    public interface IThreadSleep
+    {
+        void Sleep(int milliseconds);
+    }
+}

src/GitVersionCore/Helpers/OperationWithExponentialBackoff.cs

@@ -0,0 +1,55 @@
+using System;
+using System.Collections.Generic;
+
+namespace GitVersion.Helpers
+{
+    internal class OperationWithExponentialBackoff<T> where T : Exception
+    {
+        private IThreadSleep ThreadSleep;
+        private Action Operation;
+        private int MaxRetries;
+
+        public OperationWithExponentialBackoff(IThreadSleep threadSleep, Action operation, int maxRetries = 5)
+        {
+            if (threadSleep == null)
+                throw new ArgumentNullException("threadSleep");
+            if (maxRetries < 0)
+                throw new ArgumentOutOfRangeException("maxRetries");
+
+            this.ThreadSleep = threadSleep;
+            this.Operation = operation;
+            this.MaxRetries = maxRetries;
+        }
+
+        public void Execute()
+        {
+            var exceptions = new List<Exception>();
+
+            int tries = 0;
+            int sleepMSec = 500;
+
+            while (tries <= MaxRetries)
+            {
+                tries++;
+
+                try
+                {
+                    Operation();
+                    break;
+                }
+                catch (T e)
+                {
+                    exceptions.Add(e);
+                    if (tries > MaxRetries)
+                    {
+                        throw new AggregateException("Operation failed after maximum number of retries were exceeded.", exceptions);
+                    }
+                }
+
+                Logger.WriteInfo(string.Format("Operation failed, retrying in {0} milliseconds.", sleepMSec));
+                ThreadSleep.Sleep(sleepMSec);
+                sleepMSec *= 2;
+            }
+        }
+    }
+}

src/GitVersionCore/Helpers/ThreadSleep.cs

@@ -0,0 +1,12 @@
+namespace GitVersion.Helpers
+{
+    using System.Threading;
+
+    internal class ThreadSleep : IThreadSleep
+    {
+        public void Sleep(int milliseconds)
+        {
+            Thread.Sleep(milliseconds);
+        }
+    }
+}