Retry only for a limited number of times on encountering inaccessible file.

Author: levitation

AsyncToSyncConverter.cs

@@ -127,8 +127,14 @@ public static async Task AsyncFileUpdated(Context context)
             {
                 string fileData;
                 try
-                { 
-                    fileData = await FileExtensions.ReadAllTextAsync(Extensions.GetLongPath(context.Event.FullName), context.Token);
+                {
+                    long maxFileSize = Global.MaxFileSizeMB * (1024 * 1024);
+                    fileData = await FileExtensions.ReadAllTextAsync(Extensions.GetLongPath(context.Event.FullName), context.Token, maxFileSize: maxFileSize, retryCount: Global.RetryCountOnSrcFileOpenError);
+
+                    if (fileData == null)
+                    {
+                        await ConsoleWatch.AddMessage(ConsoleColor.Red, $"Error synchronising updates from file {context.Event.FullName}", context);
+                    }
                 }
                 catch (FileNotFoundException)   //file was removed by the time queue processing got to it
                 {

BinaryFileExtensions.cs

@@ -17,22 +17,24 @@ namespace AsyncToSyncCodeRoundtripSynchroniserMonitor
 {
     public static partial class FileExtensions
     {
-        public static int MaxByteArraySize = 0x7FFFFFC7; //https://docs.microsoft.com/en-us/dotnet/framework/configure-apps/file-schema/runtime/gcallowverylargeobjects-element?redirectedfrom=MSDN#remarks
+        public static long MaxByteArraySize = 0x7FFFFFC7; //https://docs.microsoft.com/en-us/dotnet/framework/configure-apps/file-schema/runtime/gcallowverylargeobjects-element?redirectedfrom=MSDN#remarks
 
         //https://stackoverflow.com/questions/18472867/checking-equality-for-two-byte-arrays/
         public static bool BinaryEqual(Binary a, Binary b)
         {
             return a.Equals(b);
         }
 
-        public static async Task<Tuple<byte[], long>> ReadAllBytesAsync(string path, CancellationToken cancellationToken = default(CancellationToken), long maxFileSize = 0)
+        public static async Task<Tuple<byte[], long>> ReadAllBytesAsync(string path, CancellationToken cancellationToken = default(CancellationToken), long maxFileSize = 0, int retryCount = 0)
         {
             if (path == null)
                 throw new ArgumentNullException(nameof(path));
             if (path.Length == 0)
                 throw new ArgumentException("Argument_EmptyPath: {0}", nameof(path));
 
-            while (true)
+
+            retryCount = Math.Max(0, retryCount);
+            for (int i = -1; i < retryCount; i++)
             {
                 if (cancellationToken.IsCancellationRequested)
                     return await Task.FromCanceled<Tuple<byte[], long>>(cancellationToken);
@@ -67,21 +69,19 @@ ex is IOException
                 )
                 {
                     //retry after delay
-                    try
+
+                    if (i + 1 < retryCount)     //do not sleep after last try
                     {
 #if !NOASYNC
                         await Task.Delay(1000, cancellationToken);     //TODO: config file?
 #else
                         cancellationToken.WaitHandle.WaitOne(1000);
 #endif
                     }
-                    catch (TaskCanceledException)
-                    {
-                        //do nothing here
-                        return await Task.FromCanceled<Tuple<byte[], long>>(cancellationToken);
-                    }
                 }
             }
+
+            return new Tuple<byte[], long>(null, -1);
         }
 
         public static async Task WriteAllBytesAsync(string path, byte[] contents, bool createTempFileFirst, CancellationToken cancellationToken = default(CancellationToken), int writeBufferKB = 0, int bufferWriteDelayMs = 0)
@@ -147,6 +147,7 @@ ex is IOException
                 catch (IOException)
                 {
                     //retry after delay
+
 #if !NOASYNC
                     await Task.Delay(1000, cancellationToken);     //TODO: config file?
 #else

FileExtensions.cs

@@ -20,6 +20,8 @@ namespace AsyncToSyncCodeRoundtripSynchroniserMonitor
 {
     public static partial class FileExtensions
     {
+        public static long MaxCharArraySize = 0x7FEFFFFF; //https://docs.microsoft.com/en-us/dotnet/framework/configure-apps/file-schema/runtime/gcallowverylargeobjects-element?redirectedfrom=MSDN#remarks
+
         //adapted from https://github.com/dotnet/runtime/blob/5ddc873d9ea6cd4bc6a935fec3057fe89a6932aa/src/libraries/System.IO.FileSystem/src/System/IO/File.cs
 
         //internal const int DefaultBufferSize = 4096;
@@ -41,10 +43,10 @@ private static StreamReader AsyncStreamReader(string path, Encoding encoding)
             return new StreamReader(stream, encoding, detectEncodingFromByteOrderMarks: true);
         }
 
-        public static Task<string> ReadAllTextAsync(string path, CancellationToken cancellationToken = default(CancellationToken))
-            => ReadAllTextAsync(path, UTF8NoBOM, cancellationToken);
+        public static Task<string> ReadAllTextAsync(string path, CancellationToken cancellationToken = default(CancellationToken), long maxFileSize = 0, int retryCount = 0)
+            => ReadAllTextAsync(path, UTF8NoBOM, cancellationToken, maxFileSize, retryCount);
 
-        public static async Task<string> ReadAllTextAsync(string path, Encoding encoding, CancellationToken cancellationToken = default(CancellationToken))
+        public static async Task<string> ReadAllTextAsync(string path, Encoding encoding, CancellationToken cancellationToken = default(CancellationToken), long maxFileSize = 0, int retryCount = 0)
         {
             if (path == null)
                 throw new ArgumentNullException(nameof(path));
@@ -53,38 +55,38 @@ private static StreamReader AsyncStreamReader(string path, Encoding encoding)
             if (path.Length == 0)
                 throw new ArgumentException("Argument_EmptyPath: {0}", nameof(path));
 
-            while (true)    //roland
+
+            retryCount = Math.Max(0, retryCount);
+            for (int i = -1; i < retryCount; i++)   //roland
             {
                 try    //roland
                 {
                     return cancellationToken.IsCancellationRequested
                         ? await Task.FromCanceled<string>(cancellationToken)
-                        : await InternalReadAllTextAsync(path, encoding, cancellationToken);
+                        : await InternalReadAllTextAsync(path, encoding, cancellationToken, maxFileSize);
                 }
                 catch (Exception ex) when (     //roland
                     ex is IOException
                     || ex is UnauthorizedAccessException    //can happen when a folder was just created     //TODO: abandon retries after a certain number of attempts in this case
                 )
                 {
                     //retry after delay
-                    try
+
+                    if (i + 1 < retryCount)     //do not sleep after last try
                     {
 #if !NOASYNC
                         await Task.Delay(1000, cancellationToken);     //TODO: config file?
 #else
                         cancellationToken.WaitHandle.WaitOne(1000);
 #endif
                     }
-                    catch (TaskCanceledException)
-                    {
-                        //do nothing here
-                        return await Task.FromCanceled<string>(cancellationToken);
-                    }
                 }
             }
+
+            return null;
         }
 
-        private static async Task<string> InternalReadAllTextAsync(string path, Encoding encoding, CancellationToken cancellationToken)
+        private static async Task<string> InternalReadAllTextAsync(string path, Encoding encoding, CancellationToken cancellationToken, long maxFileSize = 0)
         {
             Debug.Assert(!string.IsNullOrEmpty(path));
             Debug.Assert(encoding != null);
@@ -94,6 +96,20 @@ private static async Task<string> InternalReadAllTextAsync(string path, Encoding
             try
             {
                 cancellationToken.ThrowIfCancellationRequested();
+
+
+                if (maxFileSize > 0)
+                { 
+                    long len = sr.BaseStream.Length;    //NB! the length might change during the code execution, so need to save it into separate variable
+
+                    maxFileSize = Math.Min(MaxCharArraySize * sizeof(char), maxFileSize);
+                    if (len > maxFileSize)
+                    {
+                        return null;    //TODO: return file size so that error message can report it
+                    }
+                }
+
+
 #if NETSTANDARD
                 buffer = ArrayPool<char>.Shared.Rent(sr.CurrentEncoding.GetMaxCharCount(DefaultBufferSize));
 #else 

Program.cs

@@ -40,6 +40,8 @@ internal static class Global
         public static bool UseIdlePriority = false;
         public static bool ShowErrorAlerts = true;
 
+        public static int RetryCountOnSrcFileOpenError = 10;
+
         public static long MaxFileSizeMB = 2048;
 
 
@@ -152,6 +154,7 @@ private static void Main()
 
 
             Global.MaxFileSizeMB = fileConfig.GetLong("MaxFileSizeMB") ?? Global.MaxFileSizeMB;
+            Global.RetryCountOnSrcFileOpenError = (int?)fileConfig.GetLong("RetryCountOnSrcFileOpenError") ?? Global.RetryCountOnSrcFileOpenError;
 
 
             Global.Bidirectional = fileConfig.GetTextUpper("Bidirectional") != "FALSE";   //default is true
@@ -257,7 +260,7 @@ private static async Task MainTask()
                     watch.Start();
 
 
-                    var messageContext = new Context(
+                    var initialSyncMessageContext = new Context(
                         eventObj: null,
                         token: Global.CancellationToken.Token,
                         isSyncPath: false,   //unused here
@@ -267,16 +270,16 @@ private static async Task MainTask()
 
                     BackgroundTaskManager.Run(async () =>
                     {
-                        await ConsoleWatch.AddMessage(ConsoleColor.White, "Doing initial synchronisation...", messageContext);
+                        await ConsoleWatch.AddMessage(ConsoleColor.White, "Doing initial synchronisation...", initialSyncMessageContext);
 
-                        await ScanFolders(isInitialScan: true);
+                        await ScanFolders(initialSyncMessageContext: initialSyncMessageContext);
 
                         BackgroundTaskManager.Run(async () =>
                         {
                             await InitialSyncCountdownEvent.WaitAsync(Global.CancellationToken.Token);
 
                             //if (!Global.CancellationToken.IsCancellationRequested)
-                                await ConsoleWatch.AddMessage(ConsoleColor.White, "Done initial synchronisation...", messageContext);
+                                await ConsoleWatch.AddMessage(ConsoleColor.White, "Done initial synchronisation...", initialSyncMessageContext);
                         });
 
                     });     //BackgroundTaskManager.Run(async () =>
@@ -306,27 +309,27 @@ private static async Task MainTask()
 
         private static readonly AsyncCountdownEvent InitialSyncCountdownEvent = new AsyncCountdownEvent(1);
 
-        private static async Task ScanFolders(bool isInitialScan)
+        private static async Task ScanFolders(Context initialSyncMessageContext)
         {
             //1. Do initial synchronisation from sync to async folder   //TODO: config for enabling and ordering of this operation
-            await ScanFolder(Global.SyncPath, "*.*", isInitialScan: isInitialScan);     //NB! use *.* in order to sync resx files also
+            await ScanFolder(Global.SyncPath, "*.*", initialSyncMessageContext: initialSyncMessageContext);     //NB! use *.* in order to sync resx files also
 
             if (Global.Bidirectional)
             {
                 //2. Do initial synchronisation from async to sync folder   //TODO: config for enabling and ordering of this operation
-                await ScanFolder(Global.AsyncPath, "*.*", isInitialScan: isInitialScan);     //NB! use *.* in order to sync resx files also
+                await ScanFolder(Global.AsyncPath, "*.*", initialSyncMessageContext: initialSyncMessageContext);     //NB! use *.* in order to sync resx files also
             }
 
-            if (isInitialScan)
+            if (initialSyncMessageContext?.IsInitialScan == true)
                 InitialSyncCountdownEvent.Signal();
         }
 
-        private static async Task ScanFolder(string path, string extension, bool isInitialScan)
+        private static async Task ScanFolder(string path, string extension, Context initialSyncMessageContext)
         {
-            var fileInfos = ProcessSubDirs(new DirectoryInfo(Extensions.GetLongPath(path)), extension);
+            var fileInfos = ProcessSubDirs(new DirectoryInfo(Extensions.GetLongPath(path)), extension, initialSyncMessageContext: initialSyncMessageContext);
             await fileInfos.ForEachAsync(fileInfo => 
             {
-                if (isInitialScan)
+                if (initialSyncMessageContext?.IsInitialScan == true)
                     InitialSyncCountdownEvent.AddCount();
 
                 BackgroundTaskManager.Run(async () => 
@@ -335,19 +338,25 @@ await ConsoleWatch.OnAddedAsync
                     (
                         new DummyFileSystemEvent(fileInfo),
                         Global.CancellationToken.Token,
-                        isInitialScan
+                        initialSyncMessageContext?.IsInitialScan == true
                     );
 
-                    if (isInitialScan)
+                    if (initialSyncMessageContext?.IsInitialScan == true)
                         InitialSyncCountdownEvent.Signal();
                 });
             });
         }
 
-        private static IAsyncEnumerable<FileInfo> ProcessSubDirs(DirectoryInfo srcDirInfo, string searchPattern, int recursionLevel = 0)
+        private static IAsyncEnumerable<FileInfo> ProcessSubDirs(DirectoryInfo srcDirInfo, string searchPattern, int recursionLevel = 0, Context initialSyncMessageContext = null)
         {
             return new AsyncEnumerable<FileInfo>(async yield => {
 
+#if DEBUG && false
+                if (initialSyncMessageContext?.IsInitialScan == true)
+                    await ConsoleWatch.AddMessage(ConsoleColor.Blue, "Scanning folder " + Extensions.GetLongPath(srcDirInfo.FullName), initialSyncMessageContext);
+#endif
+
+
 #if false //this built-in functio will throw IOException in case some subfolder is an invalid reparse point
                 return new DirectoryInfo(sourceDir)
                     .GetFiles(searchPattern, SearchOption.AllDirectories);
@@ -417,7 +426,7 @@ private static IAsyncEnumerable<FileInfo> ProcessSubDirs(DirectoryInfo srcDirInf
                     }
 
 
-                    var subDirFileInfos = ProcessSubDirs(dirInfo, searchPattern, recursionLevel + 1);
+                    var subDirFileInfos = ProcessSubDirs(dirInfo, searchPattern, recursionLevel + 1, initialSyncMessageContext: initialSyncMessageContext);
                     await subDirFileInfos.ForEachAsync(async subDirFileInfo => 
                     {
                         await yield.ReturnAsync(subDirFileInfo);
@@ -926,10 +935,13 @@ public static async Task FileUpdated(Context context)
                             Tuple<byte[], long> fileDataTuple = null;
                             try
                             { 
-                                fileDataTuple = await FileExtensions.ReadAllBytesAsync(Extensions.GetLongPath(context.Event.FullName), context.Token);
+                                fileDataTuple = await FileExtensions.ReadAllBytesAsync(Extensions.GetLongPath(context.Event.FullName), context.Token, retryCount: Global.RetryCountOnSrcFileOpenError);
                                 if (fileDataTuple.Item1 == null)   //maximum length exceeded
                                 {
-                                    await AddMessage(ConsoleColor.Red, $"Error synchronising updates from file {context.Event.FullName} : fileLength > maxFileSize : {fileDataTuple.Item2} > {maxFileSize}", context);
+                                    if (fileDataTuple.Item2 >= 0)
+                                        await AddMessage(ConsoleColor.Red, $"Error synchronising updates from file {context.Event.FullName} : fileLength > maxFileSize : {fileDataTuple.Item2} > {maxFileSize}", context);
+                                    else
+                                        await AddMessage(ConsoleColor.Red, $"Error synchronising updates from file {context.Event.FullName}", context);
 
                                     return; //TODO: log error?
                                 }                            

SyncToAsyncConverter.cs

@@ -103,7 +103,13 @@ public static async Task SyncFileUpdated(Context context)
                 string fileData;
                 try
                 {
-                    fileData = await FileExtensions.ReadAllTextAsync(Extensions.GetLongPath(context.Event.FullName), context.Token);
+                    long maxFileSize = Global.MaxFileSizeMB * (1024 * 1024);
+                    fileData = await FileExtensions.ReadAllTextAsync(Extensions.GetLongPath(context.Event.FullName), context.Token, maxFileSize: maxFileSize, retryCount: Global.RetryCountOnSrcFileOpenError);
+
+                    if (fileData == null)
+                    {
+                        await ConsoleWatch.AddMessage(ConsoleColor.Red, $"Error synchronising updates from file {context.Event.FullName}", context);
+                    }
                 }
                 catch (FileNotFoundException)   //file was removed by the time queue processing got to it
                 {

appsettings.example.json

@@ -4,6 +4,7 @@
         "UseIdlePriority": false,
         "ShowErrorAlerts": true,
         "MaxFileSizeMB": 2048,
+        "RetryCountOnSrcFileOpenError": 10,
 
 
         "Bidirectional": true,