Skip to content

Commit 33ccf3f

Browse files
MathiasVPMathiasVP
authored
Merge pull request #82 from microsoft/powershell-index-files-options
PS: More fine-grained file indexing support
2 parents 22a30ab + 98a098c commit 33ccf3f

File tree

5 files changed

+99
-30
lines changed

5 files changed

+99
-30
lines changed

‎powershell/extractor/Semmle.Extraction.PowerShell.Standalone/Options.cs

Lines changed: 59 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,8 @@
1-
using Semmle.Util.Logging;
21
using System.Collections.Generic;
32
using System.IO;
43
using System.Linq;
54
using Semmle.Util;
5+
using Semmle.Util.Logging;
66

77
namespace Semmle.Extraction.PowerShell.Standalone
88
{
@@ -36,22 +36,31 @@ public override bool HandleOption(string key, string value)
3636
case "exclude":
3737
Excludes.Add(value);
3838
return true;
39+
case "file-list":
40+
Files = File.ReadAllLines(value).Select(f => new FileInfo(f)).ToArray();
41+
return true;
3942
default:
4043
return base.HandleOption(key, value);
4144
}
4245
}
4346

4447
public override bool HandleArgument(string arg)
4548
{
46-
SrcDir = arg;
47-
if (!new FileInfo(SrcDir).Exists)
49+
if (!new FileInfo(arg).Exists)
4850
{
49-
var di = new DirectoryInfo(SrcDir);
51+
var di = new DirectoryInfo(arg);
5052
if (!di.Exists)
5153
{
52-
System.Console.WriteLine("Error: The file or directory {0} does not exist", di.FullName);
54+
System.Console.WriteLine(
55+
"Error: The file or directory {0} does not exist",
56+
di.FullName
57+
);
5358
Errors = true;
5459
}
60+
else
61+
{
62+
Files = di.GetFiles("*.*", SearchOption.AllDirectories);
63+
}
5564
}
5665
return true;
5766
}
@@ -70,12 +79,48 @@ public override void InvalidArgument(string argument)
7079
/// <summary>
7180
/// Files/patterns to exclude.
7281
/// </summary>
73-
public IList<string> Excludes { get; } = new List<string>() { "node_modules", "bower_components" };
82+
public IList<string> Excludes { get; } =
83+
new List<string>() { "node_modules", "bower_components" };
84+
85+
/// <summary>
86+
/// Returns a FileInfo object for each file in the given path (recursively).
87+
/// </summary>
88+
private static FileInfo[] GetFiles(string path)
89+
{
90+
var di = new DirectoryInfo(path);
91+
return di.Exists
92+
? di.GetFiles("*.*", SearchOption.AllDirectories)
93+
: new FileInfo[] { new FileInfo(path) };
94+
}
95+
96+
/// <summary>
97+
/// Returns a list of files to extract. By default, this is the list of all files in
98+
/// the current directory. However, if the LGTM_INDEX_INCLUDE environment variable is
99+
/// set, it is used as a list of files to include instead of the files from the current
100+
/// directory.
101+
/// </summary>
102+
private static FileInfo[] GetDefaultFiles()
103+
{
104+
// Check if the LGTM_INDEX_INCLUDE environmen variable exists
105+
var include = System.Environment.GetEnvironmentVariable("LGTM_INDEX_INCLUDE");
106+
if (include != null)
107+
{
108+
System.Console.WriteLine("Using LGTM_INDEX_INCLUDE: {0}", include);
109+
return include.Split(';').Select(GetFiles).SelectMany(f => f).ToArray();
110+
}
111+
else
112+
{
113+
return new DirectoryInfo(Directory.GetCurrentDirectory()).GetFiles(
114+
"*.*",
115+
SearchOption.AllDirectories
116+
);
117+
}
118+
}
74119

75120
/// <summary>
76121
/// The directory or file containing the source code;
77122
/// </summary>
78-
public string SrcDir { get; set; } = Directory.GetCurrentDirectory();
123+
public FileInfo[] Files { get; set; } = GetDefaultFiles();
79124

80125
/// <summary>
81126
/// Whether the extraction phase should be skipped (dry-run).
@@ -109,19 +154,20 @@ public bool ExcludesFile(string path)
109154
/// </summary>
110155
public static void ShowHelp(System.IO.TextWriter output)
111156
{
112-
output.WriteLine("PowerShell# standalone extractor\n\nExtracts PowerShell scripts in the current directory.\n");
157+
output.WriteLine(
158+
"PowerShell# standalone extractor\n\nExtracts PowerShell scripts in the current directory.\n"
159+
);
113160
output.WriteLine("Additional options:\n");
114161
output.WriteLine(" <path> Use the provided path instead.");
115-
output.WriteLine(" --exclude:xxx Exclude a file or directory (can be specified multiple times)");
162+
output.WriteLine(
163+
" --exclude:xxx Exclude a file or directory (can be specified multiple times)"
164+
);
116165
output.WriteLine(" --dry-run Stop before extraction");
117166
output.WriteLine(" --threads:nnn Specify number of threads (default=CPU cores)");
118167
output.WriteLine(" --verbose Produce more output");
119-
120168
}
121169

122-
private Options()
123-
{
124-
}
170+
private Options() { }
125171

126172
public static Options Create(string[] args)
127173
{

‎powershell/extractor/Semmle.Extraction.PowerShell.Standalone/Program.cs

Lines changed: 30 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -1,13 +1,12 @@
11
using System;
22
using System.Collections.Generic;
3-
using Semmle.Util.Logging;
43
using System.IO;
54
using System.Linq;
65
using Semmle.Extraction.PowerShell;
6+
using Semmle.Util.Logging;
77

88
namespace Semmle.Extraction.PowerShell.Standalone
99
{
10-
1110
/// <summary>
1211
/// One independent run of the extractor.
1312
/// </summary>
@@ -45,9 +44,13 @@ public static int Main(string[] args)
4544
var start = DateTime.Now;
4645

4746
output.Log(Severity.Info, "Running PowerShell standalone extractor");
48-
var di = new DirectoryInfo(options.SrcDir);
49-
var sourceFiles = di.GetFiles("*.*", SearchOption.AllDirectories)
50-
.Where(d => options.Extensions.Contains(d.Extension,StringComparer.InvariantCultureIgnoreCase))
47+
var sourceFiles = options
48+
.Files.Where(d =>
49+
options.Extensions.Contains(
50+
d.Extension,
51+
StringComparer.InvariantCultureIgnoreCase
52+
)
53+
)
5154
.Select(d => d.FullName)
5255
.Where(d => !options.ExcludesFile(d))
5356
.ToArray();
@@ -64,7 +67,10 @@ public static int Main(string[] args)
6467

6568
if (!options.SkipExtraction)
6669
{
67-
using var fileLogger = new FileLogger(options.Verbosity, PowerShell.Extractor.Extractor.GetPowerShellLogPath());
70+
using var fileLogger = new FileLogger(
71+
options.Verbosity,
72+
PowerShell.Extractor.Extractor.GetPowerShellLogPath()
73+
);
6874

6975
output.Log(Severity.Info, "");
7076
output.Log(Severity.Info, "Extracting...");
@@ -73,7 +79,8 @@ public static int Main(string[] args)
7379
sourceFiles,
7480
new ExtractionProgress(output),
7581
fileLogger,
76-
options);
82+
options
83+
);
7784
output.Log(Severity.Info, $"Extraction completed in {DateTime.Now - start}");
7885
}
7986

@@ -89,14 +96,27 @@ public ExtractionProgress(ILogger output)
8996

9097
private readonly ILogger logger;
9198

92-
public void Analysed(int item, int total, string source, string output, TimeSpan time, AnalysisAction action)
99+
public void Analysed(
100+
int item,
101+
int total,
102+
string source,
103+
string output,
104+
TimeSpan time,
105+
AnalysisAction action
106+
)
93107
{
94-
logger.Log(Severity.Info, "[{0}/{1}] {2} ({3})", item, total, source,
108+
logger.Log(
109+
Severity.Info,
110+
"[{0}/{1}] {2} ({3})",
111+
item,
112+
total,
113+
source,
95114
action == AnalysisAction.Extracted
96115
? time.ToString()
97116
: action == AnalysisAction.Excluded
98117
? "excluded"
99-
: "up to date");
118+
: "up to date"
119+
);
100120
}
101121
}
102122
}

‎powershell/tools/index.cmd renamed to ‎powershell/tools/index-files.cmd

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@ if not defined CODEQL_POWERSHELL_EXTRACTOR (
44
set CODEQL_POWERSHELL_EXTRACTOR=Semmle.Extraction.PowerShell.Standalone.exe
55
)
66

7-
type NUL && "%CODEQL_EXTRACTOR_POWERSHELL_ROOT%/tools/%CODEQL_PLATFORM%/%CODEQL_POWERSHELL_EXTRACTOR%"
7+
type NUL && "%CODEQL_EXTRACTOR_POWERSHELL_ROOT%/tools/%CODEQL_PLATFORM%/%CODEQL_POWERSHELL_EXTRACTOR%" ^
8+
--file-list "%1"
89

910
exit /b %ERRORLEVEL%

‎powershell/tools/index.sh renamed to ‎powershell/tools/index-files.sh

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,4 +4,4 @@ if [[ -z "${CODEQL_EXTRACTOR_POWERSHELL_ROOT}" ]]; then
44
export CODEQL_EXTRACTOR_POWERSHELL_ROOT="Semmle.Extraction.PowerShell.Standalone.exe"
55
fi
66

7-
"$CODEQL_EXTRACTOR_POWERSHELL_ROOT/tools/$CODEQL_PLATFORM/$CODEQL_POWERSHELL_EXTRACTOR"
7+
"$CODEQL_EXTRACTOR_POWERSHELL_ROOT/tools/$CODEQL_PLATFORM/$CODEQL_POWERSHELL_EXTRACTOR" --file-list "%1"

‎powershell/tools/qltest.cmd

Lines changed: 7 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,11 @@
11
@echo off
22

3-
if not defined CODEQL_POWERSHELL_EXTRACTOR (
4-
set CODEQL_POWERSHELL_EXTRACTOR=Semmle.Extraction.PowerShell.Standalone.exe
5-
)
6-
7-
type NUL && "%CODEQL_EXTRACTOR_POWERSHELL_ROOT%/tools/%CODEQL_PLATFORM%/%CODEQL_POWERSHELL_EXTRACTOR%"
3+
type NUL && "%CODEQL_DIST%\codeql.exe" database index-files ^
4+
--prune=**/*.testproj ^
5+
--include-extension=.ps1 ^
6+
--size-limit=5m ^
7+
--language=powershell ^
8+
--working-dir=. ^
9+
"%CODEQL_EXTRACTOR_POWERSHELL_WIP_DATABASE%"
810

911
exit /b %ERRORLEVEL%

0 commit comments

Comments
 (0)