Skip to content

Commit 993dd76

Browse files
tamasvajktamasvajk
committed
C#: Add paths/paths-ignore support in standalone
1 parent 21229b9 commit 993dd76

File tree

5 files changed

+330
-1
lines changed

5 files changed

+330
-1
lines changed

‎csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DependencyManager.cs

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -439,6 +439,7 @@ private IEnumerable<FileInfo> GetAllFiles()
439439
files = files.Where(f => !f.FullName.StartsWith(options.DotNetPath, StringComparison.OrdinalIgnoreCase));
440440
}
441441

442+
files = new FilePathFilter(sourceDir, progressMonitor).Filter(files);
442443
return files;
443444
}
444445

‎csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/FilePathFilter.cs

Lines changed: 81 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,81 @@
1+
using System;
2+
using System.Collections.Generic;
3+
using System.IO;
4+
using System.Linq;
5+
using System.Security.Cryptography.X509Certificates;
6+
using System.Text.RegularExpressions;
7+
using Semmle.Util;
8+
using Semmle.Util.Logging;
9+
10+
namespace Semmle.Extraction.CSharp.DependencyFetching
11+
{
12+
public class FilePathFilter
13+
{
14+
private readonly string rootFolder;
15+
private readonly IProgressMonitor progressMonitor;
16+
17+
public FilePathFilter(DirectoryInfo sourceDir, IProgressMonitor progressMonitor)
18+
{
19+
rootFolder = FileUtils.ConvertToUnix(sourceDir.FullName.ToLowerInvariant());
20+
this.progressMonitor = progressMonitor;
21+
}
22+
23+
private class FileInclusion(string path, bool include)
24+
{
25+
public string Path { get; } = path;
26+
27+
public bool Include { get; set; } = include;
28+
}
29+
30+
private record class PathFilter(Regex Regex, bool Include);
31+
32+
public IEnumerable<FileInfo> Filter(IEnumerable<FileInfo> files)
33+
{
34+
var filters = (Environment.GetEnvironmentVariable("LGTM_INDEX_FILTERS") ?? string.Empty).Split(new[] { '\r', '\n' }, StringSplitOptions.RemoveEmptyEntries);
35+
if (filters.Length == 0)
36+
{
37+
return files;
38+
}
39+
40+
var pathFilters = new List<PathFilter>();
41+
42+
foreach (var filter in filters)
43+
{
44+
bool include;
45+
string filterText;
46+
if (filter.StartsWith("include:"))
47+
{
48+
include = true;
49+
filterText = filter.Substring("include:".Length);
50+
}
51+
else if (filter.StartsWith("exclude:"))
52+
{
53+
include = false;
54+
filterText = filter.Substring("exclude:".Length);
55+
}
56+
else
57+
{
58+
progressMonitor.Log(Severity.Info, $"Invalid filter: {filter}");
59+
continue;
60+
}
61+
62+
pathFilters.Add(new PathFilter(new Regex(new FilePattern(filterText).RegexPattern, RegexOptions.IgnoreCase | RegexOptions.Compiled | RegexOptions.Singleline), include));
63+
}
64+
65+
var fileIndex = files.ToDictionary(f => f, f => new FileInclusion(FileUtils.ConvertToUnix(f.FullName.ToLowerInvariant()).Replace(rootFolder, string.Empty).TrimStart('/'), pathFilters.All(f => !f.Include)));
66+
67+
foreach (var pathFilter in pathFilters.OrderBy(pf => pf.Include ? 0 : 1))
68+
{
69+
foreach (var file in fileIndex)
70+
{
71+
if (pathFilter.Regex.IsMatch(file.Value.Path))
72+
{
73+
fileIndex[file.Key].Include = pathFilter.Include;
74+
}
75+
}
76+
}
77+
78+
return fileIndex.Where(f => f.Value.Include).Select(f => f.Key);
79+
}
80+
}
81+
}

‎csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/IProgressMonitor.cs

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
using Semmle.Util.Logging;
2+
3+
namespace Semmle.Extraction.CSharp.DependencyFetching
4+
{
5+
public interface IProgressMonitor
6+
{
7+
void Log(Severity severity, string message);
8+
}
9+
}

‎csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/ProgressMonitor.cs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@
33

44
namespace Semmle.Extraction.CSharp.DependencyFetching
55
{
6-
internal class ProgressMonitor
6+
internal class ProgressMonitor : IProgressMonitor
77
{
88
private readonly ILogger logger;
99

‎csharp/extractor/Semmle.Extraction.Tests/FilePathFilter.cs

Lines changed: 238 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,238 @@
1+
using Xunit;
2+
using Semmle.Extraction.CSharp.DependencyFetching;
3+
using Semmle.Util.Logging;
4+
using System.Linq;
5+
using System.Collections.Generic;
6+
using System.IO;
7+
using System;
8+
9+
namespace Semmle.Extraction.Tests
10+
{
11+
public class FilePathFilterTest
12+
{
13+
private class ProgressMonitorStub : IProgressMonitor
14+
{
15+
public void Log(Severity severity, string message) { }
16+
}
17+
18+
private static (FilePathFilter TestSubject, IEnumerable<FileInfo> Files) TestSetup(string root, IEnumerable<string> paths)
19+
{
20+
root = GetPlatformSpecifixPath(root);
21+
paths = GetPlatformSpecifixPaths(paths);
22+
23+
var filePathFilter = new FilePathFilter(new DirectoryInfo(root), new ProgressMonitorStub());
24+
return (filePathFilter, paths.Select(p => new FileInfo(p)));
25+
}
26+
27+
private static string GetPlatformSpecifixPath(string file)
28+
{
29+
return file.Replace('/', Path.DirectorySeparatorChar);
30+
}
31+
32+
private static IEnumerable<string> GetPlatformSpecifixPaths(IEnumerable<string> files)
33+
{
34+
return files.Select(GetPlatformSpecifixPath);
35+
}
36+
37+
private static IEnumerable<FileInfo> GetExpected(IEnumerable<string> files)
38+
{
39+
files = GetPlatformSpecifixPaths(files);
40+
return files.Select(f => new FileInfo(f));
41+
}
42+
43+
private static void AssertEquivalence(IEnumerable<FileInfo>? expected, IEnumerable<FileInfo>? actual)
44+
{
45+
Assert.Equivalent(expected?.Select(f => f.FullName), actual?.Select(f => f.FullName), strict: true);
46+
}
47+
48+
[Fact]
49+
public void TestNoFilter()
50+
{
51+
(var testSubject, var files) = TestSetup(
52+
"/a/b",
53+
[
54+
"/a/b/c/d/e/f.cs",
55+
"/a/b/c/d/e/g.cs",
56+
"/a/b/c/d/e/h.cs",
57+
"/a/b/c/x/y/i.cs",
58+
"/a/b/c/x/z/i.cs"
59+
]);
60+
61+
Environment.SetEnvironmentVariable("LGTM_INDEX_FILTERS", null);
62+
63+
var filtered = testSubject.Filter(files);
64+
65+
AssertEquivalence(files, filtered);
66+
}
67+
68+
[Fact]
69+
public void TestFiltersWithOnlyInclude()
70+
{
71+
(var testSubject, var files) = TestSetup(
72+
"/a/b",
73+
[
74+
"/a/b/c/d/e/f.cs",
75+
"/a/b/c/d/e/g.cs",
76+
"/a/b/c/d/e/h.cs",
77+
"/a/b/c/x/y/i.cs",
78+
"/a/b/c/x/z/i.cs"
79+
]);
80+
81+
Environment.SetEnvironmentVariable("LGTM_INDEX_FILTERS", """
82+
include:c/d
83+
include:c/x/y
84+
""");
85+
86+
var filtered = testSubject.Filter(files);
87+
88+
var expected = GetExpected(
89+
[
90+
"/a/b/c/d/e/f.cs",
91+
"/a/b/c/d/e/g.cs",
92+
"/a/b/c/d/e/h.cs",
93+
"/a/b/c/x/y/i.cs"
94+
]);
95+
96+
AssertEquivalence(expected, filtered);
97+
}
98+
99+
[Fact]
100+
public void TestFiltersWithOnlyExclude()
101+
{
102+
(var testSubject, var files) = TestSetup("/a/b",
103+
[
104+
"/a/b/c/d/e/f.cs",
105+
"/a/b/c/d/e/g.cs",
106+
"/a/b/c/d/e/h.cs",
107+
"/a/b/c/x/y/i.cs",
108+
"/a/b/c/x/z/i.cs"
109+
]);
110+
111+
Environment.SetEnvironmentVariable("LGTM_INDEX_FILTERS", """
112+
exclude:c/d/e
113+
""");
114+
115+
var filtered = testSubject.Filter(files);
116+
117+
var expected = GetExpected(
118+
[
119+
"/a/b/c/x/y/i.cs",
120+
"/a/b/c/x/z/i.cs"
121+
]);
122+
123+
AssertEquivalence(expected, filtered);
124+
}
125+
126+
[Fact]
127+
public void TestFiltersWithIncludeExclude()
128+
{
129+
(var testSubject, var files) = TestSetup("/a/b",
130+
[
131+
"/a/b/c/d/e/f.cs",
132+
"/a/b/c/d/e/g.cs",
133+
"/a/b/c/d/e/h.cs",
134+
"/a/b/c/x/y/i.cs",
135+
"/a/b/c/x/z/i.cs"
136+
]);
137+
138+
Environment.SetEnvironmentVariable("LGTM_INDEX_FILTERS", """
139+
include:c/x
140+
exclude:c/x/z
141+
""");
142+
143+
var filtered = testSubject.Filter(files);
144+
145+
var expected = GetExpected(
146+
[
147+
"/a/b/c/x/y/i.cs"
148+
]);
149+
150+
AssertEquivalence(expected, filtered);
151+
}
152+
153+
[Fact]
154+
public void TestFiltersWithIncludeExcludeExcludeFirst()
155+
{
156+
(var testSubject, var files) = TestSetup("/a/b",
157+
[
158+
"/a/b/c/d/e/f.cs",
159+
"/a/b/c/d/e/g.cs",
160+
"/a/b/c/d/e/h.cs",
161+
"/a/b/c/x/y/i.cs",
162+
"/a/b/c/x/z/i.cs"
163+
]);
164+
165+
Environment.SetEnvironmentVariable("LGTM_INDEX_FILTERS", """
166+
exclude:c/x/z
167+
include:c/x
168+
""");
169+
170+
var filtered = testSubject.Filter(files);
171+
172+
var expected = GetExpected(
173+
[
174+
"/a/b/c/x/y/i.cs"
175+
]);
176+
177+
AssertEquivalence(expected, filtered);
178+
}
179+
180+
[Fact]
181+
public void TestFiltersWithIncludeExcludeComplexPatterns1()
182+
{
183+
(var testSubject, var files) = TestSetup("/a/b",
184+
[
185+
"/a/b/c/d/e/f.cs",
186+
"/a/b/c/d/e/g.cs",
187+
"/a/b/c/d/e/h.cs",
188+
"/a/b/c/x/y/i.cs",
189+
"/a/b/c/x/z/i.cs"
190+
]);
191+
192+
Environment.SetEnvironmentVariable("LGTM_INDEX_FILTERS", """
193+
include:c/**/i.*
194+
include:c/d/**/*.cs
195+
exclude:**/z/i.cs
196+
""");
197+
198+
var filtered = testSubject.Filter(files);
199+
200+
var expected = GetExpected(
201+
[
202+
"/a/b/c/d/e/f.cs",
203+
"/a/b/c/d/e/g.cs",
204+
"/a/b/c/d/e/h.cs",
205+
"/a/b/c/x/y/i.cs"
206+
]);
207+
208+
AssertEquivalence(expected, filtered);
209+
}
210+
211+
[Fact]
212+
public void TestFiltersWithIncludeExcludeComplexPatterns2()
213+
{
214+
(var testSubject, var files) = TestSetup("/a/b",
215+
[
216+
"/a/b/c/d/e/f.cs",
217+
"/a/b/c/d/e/g.cs",
218+
"/a/b/c/d/e/h.cs",
219+
"/a/b/c/x/y/i.cs",
220+
"/a/b/c/x/z/i.cs"
221+
]);
222+
223+
Environment.SetEnvironmentVariable("LGTM_INDEX_FILTERS", """
224+
include:**/i.*
225+
exclude:**/z/i.cs
226+
""");
227+
228+
var filtered = testSubject.Filter(files);
229+
230+
var expected = GetExpected(
231+
[
232+
"/a/b/c/x/y/i.cs"
233+
]);
234+
235+
AssertEquivalence(expected, filtered);
236+
}
237+
}
238+
}

0 commit comments

Comments
 (0)