Safe File Traversing (#380)

* Safe File Traversing

* Formatting Markdown

---------

Co-authored-by: Tom Longhurst <thomhurst@users.noreply.github.com>

Author: thomhurst

README.md

@@ -64,7 +64,6 @@ Define your pipeline in .NET! Strong types, intellisense, parallelisation, and t
 | ModularPipelines.WinGet | Helpers for interacting with the Windows Package Manager. | [![nuget](https://img.shields.io/nuget/v/ModularPipelines.WinGet.svg)](https://www.nuget.org/packages/ModularPipelines.WinGet/) |
 | ModularPipelines.Yarn | Helpers for interacting with Yarn CLI. | [![nuget](https://img.shields.io/nuget/v/ModularPipelines.Yarn.svg)](https://www.nuget.org/packages/ModularPipelines.Yarn/) |
 
-
 ## Getting Started
 
 If you want to see how to get started, or want to know more about ModularPipelines, [read the Documentation here](https://thomhurst.github.io/ModularPipelines)

src/ModularPipelines.Build/ReleaseNotes.md

@@ -0,0 +1 @@
+*   Improved the stability of searching nested directories for files and folders so UnauthorizedAccessException's don't occur and only return files and folders that are accessible, and doesn't break the entire enumerator

src/ModularPipelines/Engine/RequirementChecker.cs

@@ -1,3 +1,4 @@
+using System.Collections.Concurrent;
 using EnumerableAsyncProcessor.Extensions;
 using ModularPipelines.Exceptions;
 using ModularPipelines.Requirements;
@@ -17,18 +18,23 @@ public RequirementChecker(IEnumerable<IPipelineRequirement> requirements, IPipel
 
     public async Task CheckRequirementsAsync()
     {
-        var failedRequirementsNames = new List<string>();
+        var failedRequirementsNames = new ConcurrentBag<string>();
 
-        await _requirements.ToAsyncProcessorBuilder()
-            .ForEachAsync(async requirement =>
+        var groupedRequirements = _requirements.GroupBy(x => x.Order);
+
+        foreach (var pipelineRequirements in groupedRequirements)
         {
-            var requirementDecision = await requirement.MustAsync(await _moduleContextProvider.GetModuleContext());
+            await pipelineRequirements.ToAsyncProcessorBuilder()
+                .ForEachAsync(async requirement =>
+                {
+                    var requirementDecision = await requirement.MustAsync(await _moduleContextProvider.GetModuleContext());
 
-            if (!requirementDecision.Success)
-            {
-                failedRequirementsNames.Add(requirementDecision.Reason ?? requirement.GetType().Name);
-            }
-        }).ProcessInParallel();
+                    if (!requirementDecision.Success)
+                    {
+                        failedRequirementsNames.Add(requirementDecision.Reason ?? requirement.GetType().Name);
+                    }
+                }).ProcessInParallel();   
+        }
 
         if (failedRequirementsNames.Any())
         {

src/ModularPipelines/FileSystem/Folder.cs

@@ -106,12 +106,16 @@ public Folder MoveTo(string path)
 
     public File CreateFile(string name) => GetFile(name).Create();
 
-    public IEnumerable<Folder> GetFolders(Func<Folder, bool> predicate) => DirectoryInfo.EnumerateDirectories("*", SearchOption.AllDirectories)
+    public IEnumerable<Folder> GetFolders(Func<Folder, bool> predicate) => GetFolders(predicate, _ => false);
+
+    public IEnumerable<File> GetFiles(Func<File, bool> predicate) => GetFiles(predicate, _ => false);
+    
+    public IEnumerable<Folder> GetFolders(Func<Folder, bool> predicate, Func<Folder, bool> exclusionFilters) => SafeWalk.EnumerateFolders(this, exclusionFilters)
         .Select(x => new Folder(x))
         .Distinct()
         .Where(predicate);
 
-    public IEnumerable<File> GetFiles(Func<File, bool> predicate) => DirectoryInfo.EnumerateFiles("*", SearchOption.AllDirectories)
+    public IEnumerable<File> GetFiles(Func<File, bool> predicate, Func<Folder, bool> directoryExclusionFilters) => SafeWalk.EnumerateFiles(this, directoryExclusionFilters)
         .Select(x => new File(x))
         .Distinct()
         .Where(predicate);
@@ -126,9 +130,13 @@ public IEnumerable<File> GetFiles(string globPattern)
             .Distinct();
     }
 
-    public File? FindFile(Func<File, bool> predicate) => GetFiles(predicate).FirstOrDefault();
+    public File? FindFile(Func<File, bool> predicate) => FindFile(predicate, _ => false);
+
+    public Folder? FindFolder(Func<Folder, bool> predicate) => FindFolder(predicate, _ => false);
+    
+    public File? FindFile(Func<File, bool> predicate, Func<Folder, bool> directoryExclusionFilters) => GetFiles(predicate, directoryExclusionFilters).FirstOrDefault();
 
-    public Folder? FindFolder(Func<Folder, bool> predicate) => GetFolders(predicate).FirstOrDefault();
+    public Folder? FindFolder(Func<Folder, bool> predicate, Func<Folder, bool> directoryExclusionFilters) => GetFolders(predicate, directoryExclusionFilters).FirstOrDefault();
 
     public IEnumerable<File> ListFiles()
     {

src/ModularPipelines/FileSystem/SafeWalk.cs

@@ -0,0 +1,59 @@
+namespace ModularPipelines.FileSystem;
+
+internal static class SafeWalk
+{
+    public static IEnumerable<string> EnumerateFiles(Folder path, Func<Folder, bool> directoryExclusionFilters)
+    {
+        foreach (var file in Directory.EnumerateFiles(path, "*", SearchOption.TopDirectoryOnly))
+        {
+            yield return file;
+        }
+
+        var innerFiles = new List<string>();
+        foreach (var folder in Directory.EnumerateDirectories(path, "*", SearchOption.TopDirectoryOnly)
+                     .Where(x => !directoryExclusionFilters(x!)))
+        {
+            try
+            {
+                innerFiles.AddRange(EnumerateFiles(folder!, directoryExclusionFilters));
+            }
+            catch (UnauthorizedAccessException)
+            {
+                continue;
+            }
+
+            foreach (var innerFile in innerFiles)
+            {
+                yield return innerFile;
+            }
+            
+            innerFiles.Clear();
+        }
+    }
+    
+    public static IEnumerable<string> EnumerateFolders(Folder path, Func<Folder, bool> exclusionFilters)
+    {
+        var innerFolders = new List<string>();
+        foreach (var folder in Directory.EnumerateDirectories(path, "*", SearchOption.TopDirectoryOnly)
+                     .Where(x => !exclusionFilters(x!)))
+        {
+            yield return folder;
+            
+            try
+            {
+                innerFolders.AddRange(EnumerateFolders(folder!, exclusionFilters));
+            }
+            catch (UnauthorizedAccessException)
+            {
+                continue;
+            }
+            
+            foreach (var innerFolder in innerFolders)
+            {
+                yield return innerFolder;
+            }
+            
+            innerFolders.Clear();
+        }
+    }
+}

src/ModularPipelines/Requirements/IPipelineRequirement.cs

@@ -6,4 +6,6 @@ namespace ModularPipelines.Requirements;
 public interface IPipelineRequirement
 {
     Task<RequirementDecision> MustAsync(IPipelineHookContext context);
+
+    int Order => 0;
 }

test/ModularPipelines.UnitTests/FolderTests.cs

@@ -316,6 +316,29 @@ public void AssertExists_ThrowsWhenNull()
         Assert.Throws<DirectoryNotFoundException>(() => folder.AssertExists());
     }
 
+    [Test, WindowsOnlyTest]
+    public void Searching_Local_Files_User_Does_Not_Throw_Unauth_Exception()
+    {
+        Assert.DoesNotThrow(() => new Folder(Environment.GetFolderPath(Environment.SpecialFolder.UserProfile))
+            .GetFolder("AppData")
+            ?.FindFile(x => x.Name.Contains(Guid.NewGuid().ToString()), exclude => exclude.Name.StartsWith('.')));
+    }
+    
+    [Test, WindowsOnlyTest]
+    public void Searching_Local_Files_User_Does_Not_Throw_Unauth_Exception2()
+    {
+        Assert.DoesNotThrow(() => new Folder(Environment.GetFolderPath(Environment.SpecialFolder.UserProfile))
+            ?.GetFiles(Guid.NewGuid().ToString()));
+    }
+    
+    [Test, WindowsOnlyTest]
+    public void Searching_Local_Folders_User_Does_Not_Throw_Unauth_Exception()
+    {
+        Assert.DoesNotThrow(() => new Folder(Environment.GetFolderPath(Environment.SpecialFolder.UserProfile))
+                .GetFolder("AppData")
+            ?.FindFolder(x => x.Name.Contains(Guid.NewGuid().ToString()), exclude => exclude.Name.StartsWith('.')));
+    }
+
     private static Folder CreateRandomFolder()
     {
         var tempFolderPath = Path.Combine(Path.GetTempPath(), Guid.NewGuid().ToString("N"));