Skip to content

Commit 2ee323c

Browse files
rameelrameel
authored
Merge pull request #22 from rameel/cleanup
Pre-release
2 parents 9d21dad + cb39029 commit 2ee323c

File tree

5 files changed

+142
-170
lines changed

5 files changed

+142
-170
lines changed

‎README.md‎

Lines changed: 66 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -102,7 +102,8 @@ Similarly, for the `a/**/a/**/a/**/.../a/**/a/**/a/**/b` pattern matching agains
102102

103103
## File traversal
104104

105-
The `Files` class provides functionality for traversing the file system and retrieving lists of files and directories based on specified glob patterns. This allows for flexible and efficient file and directory enumeration.
105+
The `Files` class provides functionality for traversing the file system and retrieving lists of files and directories
106+
based on specified glob patterns. This allows for flexible and efficient file and directory enumeration.
106107

107108
```csharp
108109
using Ramstack.Globbing.Traversal;
@@ -128,8 +129,72 @@ foreach (var file in files)
128129
Console.WriteLine(file);
129130
```
130131

132+
There are overloads that take a `TraversalOptions` allowing you to set additional options when traversing the file system,
133+
such as:
134+
- Filtering out specific attributes
135+
- Ignoring inaccessible files
136+
- Maximum recursion depth
137+
138+
These methods are quite efficient in terms of speed, memory consumption, and GC pressure.
139+
Here are the benchmarking results for the [dotnet/runtime](https://github.com/dotnet/runtime) repository folder, which contained
140+
59194 files at the time of testing. The search was for `*.md` files:
141+
142+
```
143+
BenchmarkDotNet v0.13.12, Windows 11 (10.0.22631.3880/23H2/2023Update/SunValley3)
144+
AMD Ryzen 9 5900X, 1 CPU, 24 logical and 12 physical cores
145+
.NET SDK 9.0.100-preview.6.24328.19
146+
[Host] : .NET 8.0.7 (8.0.724.31311), X64 RyuJIT AVX2
147+
Job-GMSEBO : .NET 8.0.7 (8.0.724.31311), X64 RyuJIT AVX2
148+
149+
Runtime=.NET 8.0
150+
151+
| Method | Mean | Error | StdDev | Gen0 | Gen1 | Allocated |
152+
|----------------------------------- |---------:|--------:|--------:|----------:|---------:|----------:|
153+
| >> Ramstack_Files_EnumerateFiles | 154.4 ms | 0.67 ms | 0.59 ms | - | - | 2.33 MB |
154+
| Microsoft_Directory_EnumerateFiles | 149.1 ms | 0.74 ms | 0.66 ms | - | - | 2.33 MB |
155+
| Microsoft_FileSystemGlobbing | 176.6 ms | 1.16 ms | 1.08 ms | 2000.0000 | 333.3333 | 35.99 MB |
156+
```
157+
158+
As you can see, the code is as fast as a direct search using `Directory.EnumerateFiles` and consumes the same amount of memory.
159+
This makes sense, since the implementation of `Files.EnumerateFiles` uses the same `FileSystemEnumerable` class.
160+
161+
Also, corresponding extension methods added for `DirectoryInfo`, which also allow you to leverage
162+
the full power of glob patterns when searching for files.
163+
164+
## Custom File Systems (`FileTreeEnumerable`)
165+
166+
The `FileTreeEnumerable` class provides support for custom file systems with glob pattern matching capabilities.
167+
Here is an example of its usage:
168+
```csharp
169+
// As an example, we'll use the existing DirectoryInfo/FileInfo classes.
170+
171+
var root = new DirectoryInfo(@"D:\Projects\dotnet.runtime");
172+
var enumeration = new FileTreeEnumerable<FileSystemInfo, string>(root)
173+
{
174+
Patterns = ["**/*.cs"],
175+
Excludes = ["**/bin", "**/obj"],
176+
FileNameSelector = info => info.Name,
177+
ShouldRecursePredicate = info => info is DirectoryInfo,
178+
// The following predicate used to filter the files
179+
ShouldIncludePredicate = info => info is FileInfo,
180+
ChildrenSelector = info => ((DirectoryInfo)info).EnumerateFileSystemInfos(),
181+
// Returns the full path of the file
182+
ResultSelector = info => info.FullName
183+
};
184+
185+
// Prints all csharp files
186+
foreach (var filePath in enumeration)
187+
Console.WriteLine(filePath);
188+
```
189+
190+
131191
## Changelog
132192

193+
### 2.1.0
194+
* Added the `FileTreeEnumerable` class for custom file systems with glob pattern support.
195+
* Added overloads with `TraversalOptions` for file enumeration.
196+
* Added extension methods for `DirectoryInfo`.
197+
133198
### 2.0.0
134199
* Added the ability to retrieve a list of files and directories based on a specified glob pattern.
135200

‎Ramstack.Globbing.Tests/Traversal/FileTreeEnumerableTests.cs‎

Lines changed: 75 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,75 @@
1+
using Ramstack.Globbing.Traversal.Helpers;
2+
3+
namespace Ramstack.Globbing.Traversal;
4+
5+
[TestFixture]
6+
public class FileTreeEnumerableTests
7+
{
8+
private readonly TempFileStorage _storage = new();
9+
10+
[OneTimeTearDown]
11+
public void Cleanup() =>
12+
_storage.Dispose();
13+
14+
[Test]
15+
public void Enumerate_Files()
16+
{
17+
var enumerable = new FileTreeEnumerable<FileSystemInfo, string>(new DirectoryInfo(_storage.Root))
18+
{
19+
Patterns = ["**"],
20+
Flags = MatchFlags.Auto,
21+
FileNameSelector = info => info.Name,
22+
ShouldRecursePredicate = info => info is DirectoryInfo,
23+
ShouldIncludePredicate = info => info is FileInfo,
24+
ResultSelector = info => info.FullName,
25+
ChildrenSelector = info => ((DirectoryInfo)info).EnumerateFileSystemInfos()
26+
}.OrderBy(p => p);
27+
28+
var expected = Directory
29+
.EnumerateFiles(_storage.Root, "*", SearchOption.AllDirectories)
30+
.OrderBy(p => p);
31+
32+
Assert.That(enumerable, Is.EquivalentTo(expected));
33+
}
34+
35+
[Test]
36+
public void Enumerate_Directories()
37+
{
38+
var enumerable = new FileTreeEnumerable<FileSystemInfo, string>(new DirectoryInfo(_storage.Root))
39+
{
40+
Patterns = ["**"],
41+
Flags = MatchFlags.Auto,
42+
FileNameSelector = info => info.Name,
43+
ShouldRecursePredicate = info => info is DirectoryInfo,
44+
ShouldIncludePredicate = info => info is DirectoryInfo,
45+
ResultSelector = info => info.FullName,
46+
ChildrenSelector = info => ((DirectoryInfo)info).EnumerateFileSystemInfos()
47+
}.OrderBy(p => p);
48+
49+
var expected = Directory
50+
.EnumerateDirectories(_storage.Root, "*", SearchOption.AllDirectories)
51+
.OrderBy(p => p);
52+
53+
Assert.That(enumerable, Is.EquivalentTo(expected));
54+
}
55+
56+
[Test]
57+
public void Enumerate_Both()
58+
{
59+
var enumerable = new FileTreeEnumerable<FileSystemInfo, string>(new DirectoryInfo(_storage.Root))
60+
{
61+
Patterns = ["**"],
62+
Flags = MatchFlags.Auto,
63+
FileNameSelector = info => info.Name,
64+
ShouldRecursePredicate = info => info is DirectoryInfo,
65+
ResultSelector = info => info.FullName,
66+
ChildrenSelector = info => ((DirectoryInfo)info).EnumerateFileSystemInfos()
67+
}.OrderBy(p => p);
68+
69+
var expected = Directory
70+
.EnumerateFileSystemEntries(_storage.Root, "*", SearchOption.AllDirectories)
71+
.OrderBy(p => p);
72+
73+
Assert.That(enumerable, Is.EquivalentTo(expected));
74+
}
75+
}

‎Ramstack.Globbing.Tests/Traversal/PathHelperTests.cs‎

Lines changed: 0 additions & 39 deletions
Original file line numberDiff line numberDiff line change
@@ -154,45 +154,6 @@ public void GetPartialPattern_Generated()
154154
}
155155
}
156156

157-
[Test]
158-
public void SearchPathSeparator()
159-
{
160-
for (var n = 0; n < 5000; n++)
161-
{
162-
var p0 = new string('a', n);
163-
164-
var p1 = p0 + "\\";
165-
var index1 = PathHelper.SearchPathSeparator(p1, MatchFlags.Windows);
166-
var index2 = PathHelper.SearchPathSeparator(p1, MatchFlags.Unix);
167-
168-
Assert.That(index1, Is.EqualTo(p1.IndexOf('\\')), $"length: {n}");
169-
Assert.That(index2, Is.EqualTo(-1), $"length: {n}");
170-
171-
var p2 = p0 + "/";
172-
var index3 = PathHelper.SearchPathSeparator(p2, MatchFlags.Windows);
173-
var index4 = PathHelper.SearchPathSeparator(p2, MatchFlags.Unix);
174-
175-
Assert.That(index3, Is.EqualTo(p2.IndexOf('/')), $"length: {n}");
176-
Assert.That(index4, Is.EqualTo(index3), $"length: {n}");
177-
}
178-
}
179-
180-
[Test]
181-
public void SearchPathSeparator_Nothing()
182-
{
183-
var source = new string('a', 5000);
184-
185-
for (var n = 0; n < 5000; n++)
186-
{
187-
var p = source.AsSpan(0, n);
188-
var index1 = PathHelper.SearchPathSeparator(p, MatchFlags.Windows);
189-
var index2 = PathHelper.SearchPathSeparator(p, MatchFlags.Unix);
190-
191-
Assert.That(index1, Is.EqualTo(-1));
192-
Assert.That(index2, Is.EqualTo(-1));
193-
}
194-
}
195-
196157
private static IEnumerable<(string path, char slash, MatchFlags flags)> GeneratePaths()
197158
{
198159
var flags = new[]

‎Ramstack.Globbing/Traversal/FileTreeEnumerable.cs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -101,7 +101,7 @@ private IEnumerable<TResult> Enumerate()
101101
private static ReadOnlySpan<char> GetFullName(ref char[] chars, string path, string name)
102102
{
103103
var length = path.Length + name.Length + 1;
104-
if (chars.Length <= length)
104+
if (chars.Length < length)
105105
{
106106
ArrayPool<char>.Shared.Return(chars);
107107
chars = ArrayPool<char>.Shared.Rent(length);

‎Ramstack.Globbing/Traversal/PathHelper.cs‎

Lines changed: 0 additions & 129 deletions
Original file line numberDiff line numberDiff line change
@@ -15,135 +15,6 @@ namespace Ramstack.Globbing.Traversal;
1515
[SuppressMessage("ReSharper", "RedundantCast")]
1616
internal static class PathHelper
1717
{
18-
/// <summary>
19-
/// Searches for a path separator within a span of characters.
20-
/// </summary>
21-
/// <param name="path">The span of characters representing the path.</param>
22-
/// <param name="flags">The flags indicating the type of path separators to match.</param>
23-
/// <returns>
24-
/// The index of the path separator if found; otherwise, <c>-1</c>.
25-
/// </returns>
26-
public static int SearchPathSeparator(scoped ReadOnlySpan<char> path, MatchFlags flags)
27-
{
28-
var count = (nint)(uint)path.Length;
29-
ref var p = ref MemoryMarshal.GetReference(path);
30-
return SearchPathSeparator(ref p, count, flags);
31-
}
32-
33-
/// <summary>
34-
/// Searches for a path separator within a range of characters.
35-
/// </summary>
36-
/// <param name="p">A reference to the first character of the range.</param>
37-
/// <param name="count">The number of characters to search through.</param>
38-
/// <param name="flags">The flags indicating the type of path separators to match.</param>
39-
/// <returns>
40-
/// The index of the path separator if found; otherwise, <c>-1</c>.
41-
/// </returns>
42-
public static int SearchPathSeparator(scoped ref char p, nint count, MatchFlags flags)
43-
{
44-
var index = (nint)0;
45-
46-
if (!Sse41.IsSupported || count < Vector128<ushort>.Count)
47-
{
48-
for (; index < count; index++)
49-
if (Unsafe.Add(ref p, index) == '/'
50-
|| (Unsafe.Add(ref p, index) == '\\' && flags == MatchFlags.Windows))
51-
return (int)index;
52-
53-
return -1;
54-
}
55-
56-
if (!Avx2.IsSupported || count < Vector256<ushort>.Count)
57-
{
58-
var slash = Vector128.Create((ushort)'/');
59-
var backslash = Vector128.Create((ushort)'\\');
60-
var allowEscaping = CreateAllowEscaping128Bitmask(flags);
61-
62-
var tail = count - Vector128<ushort>.Count;
63-
64-
Vector128<ushort> chunk;
65-
Vector128<ushort> comparison;
66-
67-
do
68-
{
69-
chunk = LoadVector128(ref p, index);
70-
comparison = Sse2.Or(
71-
Sse2.CompareEqual(chunk, slash),
72-
Sse2.AndNot(
73-
allowEscaping,
74-
Sse2.CompareEqual(chunk, backslash)));
75-
76-
if (!Sse41.TestZ(comparison, comparison))
77-
{
78-
var position = BitOperations.TrailingZeroCount(Sse2.MoveMask(comparison.AsByte()));
79-
return (int)index + (position >>> 1);
80-
}
81-
82-
index += Vector128<ushort>.Count;
83-
}
84-
while (index + Vector128<ushort>.Count <= count);
85-
86-
chunk = LoadVector128(ref p, tail);
87-
comparison = Sse2.Or(
88-
Sse2.CompareEqual(chunk, slash),
89-
Sse2.AndNot(
90-
allowEscaping,
91-
Sse2.CompareEqual(chunk, backslash)));
92-
93-
if (!Sse41.TestZ(comparison, comparison))
94-
{
95-
var position = BitOperations.TrailingZeroCount(Sse2.MoveMask(comparison.AsByte()));
96-
return (int)tail + (position >>> 1);
97-
}
98-
99-
return -1;
100-
}
101-
else
102-
{
103-
var slash = Vector256.Create((ushort)'/');
104-
var backslash = Vector256.Create((ushort)'\\');
105-
var allowEscaping = CreateAllowEscaping256Bitmask(flags);
106-
var tail = count - Vector256<ushort>.Count;
107-
108-
Vector256<ushort> chunk;
109-
Vector256<ushort> comparison;
110-
111-
do
112-
{
113-
chunk = LoadVector256(ref p, index);
114-
comparison = Avx2.Or(
115-
Avx2.CompareEqual(chunk, slash),
116-
Avx2.AndNot(
117-
allowEscaping,
118-
Avx2.CompareEqual(chunk, backslash)));
119-
120-
if (!Avx.TestZ(comparison, comparison))
121-
{
122-
var position = BitOperations.TrailingZeroCount(Avx2.MoveMask(comparison.AsByte()));
123-
return (int)index + (position >>> 1);
124-
}
125-
126-
index += Vector256<ushort>.Count;
127-
}
128-
while (index + Vector256<ushort>.Count <= count);
129-
130-
chunk = LoadVector256(ref p, tail);
131-
comparison = Avx2.Or(
132-
Avx2.CompareEqual(chunk, slash),
133-
Avx2.AndNot(
134-
allowEscaping,
135-
Avx2.CompareEqual(chunk, backslash)));
136-
137-
if (!Avx.TestZ(comparison, comparison))
138-
{
139-
var position = BitOperations.TrailingZeroCount(Avx2.MoveMask(comparison.AsByte()));
140-
return (int)tail + (position >>> 1);
141-
}
142-
143-
return -1;
144-
}
145-
}
146-
14718
/// <summary>
14819
/// Determines whether the specified path matches any of the specified patterns.
14920
/// </summary>

0 commit comments

Comments
 (0)