Optimised exclusion matching. Added *IgnorePathsEndingWith settings. Require slashes in the start of *IgnorePathsStartingWith entries.

Author: levitation

AsyncToSyncCodeRoundtripSynchroniserMonitorNet.csproj

@@ -99,6 +99,9 @@
     <Reference Include="Nito.Disposables, Version=2.0.0.0, Culture=neutral, processorArchitecture=MSIL">
       <HintPath>packages\Nito.Disposables.2.0.0\lib\netstandard1.0\Nito.Disposables.dll</HintPath>
     </Reference>
+    <Reference Include="NReco.Text.AhoCorasickDoubleArrayTrie, Version=1.0.2.0, Culture=neutral, PublicKeyToken=70ca172a68680917, processorArchitecture=MSIL">
+      <HintPath>..\..\FolderSync\FolderSyncNet\packages\NReco.Text.AhoCorasickDoubleArrayTrie.1.0.2\lib\net40\NReco.Text.AhoCorasickDoubleArrayTrie.dll</HintPath>
+    </Reference>
     <Reference Include="System" />
     <Reference Include="System.AppContext, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a, processorArchitecture=MSIL">
       <HintPath>packages\System.AppContext.4.3.0\lib\net46\System.AppContext.dll</HintPath>

Extensions.cs

@@ -81,7 +81,10 @@ public static string GetLongPath(string path)
 
             //@"\\?\" prefix is needed for reading from long paths: https://stackoverflow.com/questions/44888844/directorynotfoundexception-when-using-long-paths-in-net-4-7 and https://superuser.com/questions/1617012/support-of-the-unc-server-share-syntax-in-windows
 
-            if (path.Substring(0, 2) == @"\\")   //network path or path already starting with \\?\
+            if (
+                path.Length >= 2    //necessary to avoid exceptions in Substring()
+                && path.Substring(0, 2) == @"\\"   //network path or path already starting with \\?\ 
+            )
             {
                 return path;
             }

Program.cs

@@ -25,6 +25,7 @@
 using myoddweb.directorywatcher.interfaces;
 using Nito.AspNetBackgroundTasks;
 using Nito.AsyncEx;
+using NReco.Text;
 
 namespace AsyncToSyncCodeRoundtripSynchroniserMonitor
 {
@@ -42,12 +43,17 @@ internal static class Global
         public static long MaxFileSizeMB = 2048;
 
 
-        public static List<string> WatchedCodeExtension = new List<string>() { "cs", "py" };
-        public static List<string> WatchedResXExtension = new List<string>() { "resx" };
+        public static HashSet<string> WatchedCodeExtension = new HashSet<string>() { "cs", "py" };
+        public static HashSet<string> WatchedResXExtension = new HashSet<string>() { "resx" };
 
-        public static List<string> ExcludedExtensions = new List<string>() { "*~", "tmp" };
-        public static List<string> IgnorePathsStartingWith = new List<string>();
-        public static List<string> IgnorePathsContaining = new List<string>();
+        public static HashSet<string> ExcludedExtensions = new HashSet<string>() { "*~", "tmp" };
+
+        public static List<string> IgnorePathsStartingWithList = new List<string>();
+        public static List<string> IgnorePathsContainingList = new List<string>();
+        public static List<string> IgnorePathsEndingWithList = new List<string>();
+
+        public static bool IgnorePathsContainingACHasAny = false;
+        public static AhoCorasickDoubleArrayTrie<bool> IgnorePathsContainingAC = new AhoCorasickDoubleArrayTrie<bool>();
 
         public static string AsyncPath = "";
         public static string SyncPath = "";
@@ -97,6 +103,15 @@ public bool Is(EventAction action)
 
     internal class Program
     {
+        //let null char mark start and end of a filename
+        //https://stackoverflow.com/questions/54205087/how-can-i-create-a-file-with-null-bytes-in-the-filename
+        //https://stackoverflow.com/questions/1976007/what-characters-are-forbidden-in-windows-and-linux-directory-names
+        //https://serverfault.com/questions/242110/which-common-characters-are-illegal-in-unix-and-windows-filesystems
+        public static readonly string NullChar = new string(new char[] { (char)0 });
+
+        public static readonly string DirectorySeparatorChar = new string(new char[] { Path.DirectorySeparatorChar });
+
+
         private static byte[] GetHash(string inputString)
         {
 #pragma warning disable SCS0006     //Warning	SCS0006	Weak hashing function
@@ -151,14 +166,26 @@ private static void Main()
             Global.AsyncPathMinFreeSpace = fileConfig.GetLong("AsyncPathMinFreeSpace") ?? 0;
             Global.SyncPathMinFreeSpace = fileConfig.GetLong("SyncPathMinFreeSpace") ?? 0;
 
-            Global.WatchedCodeExtension = fileConfig.GetListUpperOnWindows(Global.CaseSensitiveFilenames, "WatchedCodeExtensions", "WatchedCodeExtension");
-            Global.WatchedResXExtension = fileConfig.GetListUpperOnWindows(Global.CaseSensitiveFilenames, "WatchedResXExtensions", "WatchedResXExtension");
+            Global.WatchedCodeExtension = new HashSet<string>(fileConfig.GetListUpperOnWindows(Global.CaseSensitiveFilenames, "WatchedCodeExtensions", "WatchedCodeExtension"));
+            Global.WatchedResXExtension = new HashSet<string>(fileConfig.GetListUpperOnWindows(Global.CaseSensitiveFilenames, "WatchedResXExtensions", "WatchedResXExtension"));
 
             //this would need Microsoft.Extensions.Configuration and Microsoft.Extensions.Configuration.Binder packages
-            Global.ExcludedExtensions = fileConfig.GetListUpperOnWindows(Global.CaseSensitiveFilenames, "ExcludedExtensions", "ExcludedExtension");   //NB! UpperOnWindows
+            Global.ExcludedExtensions = new HashSet<string>(fileConfig.GetListUpperOnWindows(Global.CaseSensitiveFilenames, "ExcludedExtensions", "ExcludedExtension"));   //NB! UpperOnWindows
 
-            Global.IgnorePathsStartingWith = fileConfig.GetListUpperOnWindows(Global.CaseSensitiveFilenames, "IgnorePathsStartingWith", "IgnorePathStartingWith");   //NB! UpperOnWindows
-            Global.IgnorePathsContaining = fileConfig.GetListUpperOnWindows(Global.CaseSensitiveFilenames, "IgnorePathsContaining", "IgnorePathContaining");   //NB! UpperOnWindows
+            Global.IgnorePathsStartingWithList = fileConfig.GetListUpperOnWindows(Global.CaseSensitiveFilenames, "IgnorePathsStartingWith", "IgnorePathStartingWith");   //NB! UpperOnWindows
+            Global.IgnorePathsContainingList = fileConfig.GetListUpperOnWindows(Global.CaseSensitiveFilenames, "IgnorePathsContaining", "IgnorePathContaining");   //NB! UpperOnWindows
+            Global.IgnorePathsEndingWithList = fileConfig.GetListUpperOnWindows(Global.CaseSensitiveFilenames, "IgnorePathsEndingWith", "IgnorePathEndingWith");   //NB! UpperOnWindows
+
+            var ACInput = Global.IgnorePathsStartingWithList.Select(x => new KeyValuePair<string, bool>(NullChar + x, false))
+                .Concat(Global.IgnorePathsContainingList.Select(x => new KeyValuePair<string, bool>(x, false)))
+                .Concat(Global.IgnorePathsEndingWithList.Select(x => new KeyValuePair<string, bool>(x + NullChar, false)))
+                .ToList();
+
+            if (ACInput.Any())  //needed to avoid exceptions
+            {
+                Global.IgnorePathsContainingACHasAny = true;
+                Global.IgnorePathsContainingAC.Build(ACInput);
+            }
 
 
             var pathHashes = "";
@@ -377,10 +404,13 @@ private static IAsyncEnumerable<FileInfo> ProcessSubDirs(DirectoryInfo srcDirInf
                         continue;
 
 
-                    var nonFullNameInvariant = ConsoleWatch.GetNonFullName(dirInfo.FullName) + Path.PathSeparator;
+                    var nonFullNameInvariantWithLeadingSlash = DirectorySeparatorChar + Extensions.GetDirPathWithTrailingSlash(ConsoleWatch.GetNonFullName(dirInfo.FullName.ToUpperInvariantOnWindows(Global.CaseSensitiveFilenames)));
                     if (
-                        Global.IgnorePathsStartingWith.Any(x => nonFullNameInvariant.StartsWith(x))
-                        || Global.IgnorePathsContaining.Any(x => nonFullNameInvariant.Contains(x))
+                        //Global.IgnorePathsStartingWith.Any(x => nonFullNameInvariantWithLeadingSlash.StartsWith(x))
+                        //|| Global.IgnorePathsContaining.Any(x => nonFullNameInvariantWithLeadingSlash.Contains(x))
+                        //|| Global.IgnorePathsEndingWith.Any(x => nonFullNameInvariantWithLeadingSlash.EndsWith(x))
+                        Global.IgnorePathsContainingACHasAny  //needed to avoid exceptions
+                        && Global.IgnorePathsContainingAC.ParseText(NullChar + nonFullNameInvariantWithLeadingSlash/* + NullChar*/).Any() //NB! no NullChar appended to end since it is dir path not complete file path
                     )
                     {
                         continue;
@@ -1037,7 +1067,7 @@ private static bool IsWatchedFile(string fullName)
                     || Global.WatchedResXExtension.Contains("*")
                 )
                 &&
-                Global.ExcludedExtensions.All(x =>
+                Global.ExcludedExtensions.All(x =>  //TODO: optimise
 
                     !fullNameInvariant.EndsWith("." + x)
                     &&
@@ -1049,11 +1079,14 @@ private static bool IsWatchedFile(string fullName)
                 )
             )
             {
-                var nonFullNameInvariant = GetNonFullName(fullNameInvariant);
+                var nonFullNameInvariantWithLeadingSlash = Program.DirectorySeparatorChar + GetNonFullName(fullNameInvariant);
 
                 if (
-                    Global.IgnorePathsStartingWith.Any(x => nonFullNameInvariant.StartsWith(x))
-                    || Global.IgnorePathsContaining.Any(x => nonFullNameInvariant.Contains(x))
+                    //Global.IgnorePathsStartingWith.Any(x => nonFullNameInvariantWithLeadingSlash.StartsWith(x))
+                    //|| Global.IgnorePathsContaining.Any(x => nonFullNameInvariantWithLeadingSlash.Contains(x))
+                    //|| Global.IgnorePathsEndingWith.Any(x => nonFullNameInvariantWithLeadingSlash.EndsWith(x))
+                    Global.IgnorePathsContainingACHasAny  //needed to avoid exceptions
+                    && Global.IgnorePathsContainingAC.ParseText(Program.NullChar + nonFullNameInvariantWithLeadingSlash + Program.NullChar).Any()
                 )
                 {
                     return false;

appsettings.example.json

@@ -48,6 +48,9 @@
         "IgnorePathsContaining": [
             ".localhistory\\",
             ".vshistory\\"
+        ],
+
+        "IgnorePathsEndingWith": [
         ]
     }
 }

copyrights.txt

@@ -5,4 +5,4 @@ MyOddWeb.DirectoryWatcher 0.1.8, 2018 Florent Guelfucci, MIT Licence. https://ww
 .NET runtime source, .NET Foundation and Contributors, MIT License. https://github.com/dotnet/runtime/blob/5ddc873d9ea6cd4bc6a935fec3057fe89a6932aa/src/libraries/System.IO.FileSystem/src/System/IO/File.cs
 Nito.AspNetBackgroundTasks 1.0.0, Stephen Cleary, MIT License. https://www.nuget.org/packages/Nito.AspNetBackgroundTasks/
 Nito.AsyncEx 5.0.0, Stephen Cleary, MIT License. https://www.nuget.org/packages/Nito.AsyncEx.Coordination/
-
+NReco.Text.AhoCorasickDoubleArrayTrie 1.0.2, 2017-2019 Vitaliy Fedorchenko, Apache-2.0 License. https://www.nuget.org/packages/NReco.Text.AhoCorasickDoubleArrayTrie

packages.config

@@ -19,6 +19,7 @@
   <package id="Nito.AsyncEx.Tasks" version="5.0.0" targetFramework="net46" />
   <package id="Nito.Collections.Deque" version="1.0.4" targetFramework="net46" />
   <package id="Nito.Disposables" version="2.0.0" targetFramework="net46" />
+  <package id="NReco.Text.AhoCorasickDoubleArrayTrie" version="1.0.2" targetFramework="net46" />
   <package id="System.AppContext" version="4.3.0" targetFramework="net46" />
   <package id="System.Collections" version="4.3.0" targetFramework="net46" />
   <package id="System.Collections.Concurrent" version="4.3.0" targetFramework="net46" />