Skip to content

Commit 31ad195

Browse files
committed
C#: Add binlog support to buildless with source generator support
1 parent b2d2f2d commit 31ad195

File tree

23 files changed

+237
-42
lines changed

23 files changed

+237
-42
lines changed

csharp/autobuilder/Semmle.Autobuild.CSharp/BUILD.bazel

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@ codeql_csharp_binary(
1313
"//csharp/autobuilder/Semmle.Autobuild.Shared",
1414
"//csharp/extractor/Semmle.Extraction.CSharp",
1515
"//csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching",
16+
"//csharp/extractor/Semmle.Extraction.CSharp.Driver:bin/Semmle.Extraction.CSharp.Driver",
1617
"//csharp/extractor/Semmle.Extraction.CSharp.Standalone:bin/Semmle.Extraction.CSharp.Standalone",
1718
"//csharp/extractor/Semmle.Util",
1819
"@paket.main//microsoft.build",

csharp/autobuilder/Semmle.Autobuild.CSharp/CSharpAutobuilder.cs

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -13,8 +13,10 @@ public class CSharpAutobuildOptions : AutobuildOptionsShared
1313
{
1414
private const string buildModeEnvironmentVariable = "CODEQL_EXTRACTOR_CSHARP_BUILD_MODE";
1515
internal const string ExtractorOptionBuildless = "CODEQL_EXTRACTOR_CSHARP_OPTION_BUILDLESS";
16+
internal const string ExtractorOptionBinlog = "CODEQL_EXTRACTOR_CSHARP_OPTION_BINLOG";
1617

1718
public bool Buildless { get; }
19+
public string? Binlog { get; }
1820

1921
public override Language Language => Language.CSharp;
2022

@@ -29,7 +31,7 @@ public CSharpAutobuildOptions(IBuildActions actions) : base(actions)
2931
actions.GetEnvironmentVariable(ExtractorOptionBuildless).AsBool("buildless", false) ||
3032
actions.GetEnvironmentVariable(buildModeEnvironmentVariable)?.ToLower() == "none";
3133

32-
34+
Binlog = actions.GetEnvironmentVariable(ExtractorOptionBinlog);
3335
}
3436
}
3537

csharp/autobuilder/Semmle.Autobuild.CSharp/Semmle.Autobuild.CSharp.csproj

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@
66
<ProjectReference Include="..\..\extractor\Semmle.Util\Semmle.Util.csproj" />
77
<ProjectReference Include="..\..\extractor\Semmle.Extraction.CSharp\Semmle.Extraction.CSharp.csproj" />
88
<ProjectReference Include="..\..\extractor\Semmle.Extraction.CSharp.Standalone\Semmle.Extraction.CSharp.Standalone.csproj" />
9+
<ProjectReference Include="..\..\extractor\Semmle.Extraction.CSharp.Driver\Semmle.Extraction.CSharp.Driver.csproj" />
910
<ProjectReference Include="..\..\extractor\Semmle.Extraction.CSharp.DependencyFetching\Semmle.Extraction.CSharp.DependencyFetching.csproj" />
1011
<ProjectReference Include="..\Semmle.Autobuild.Shared\Semmle.Autobuild.Shared.csproj" />
1112
</ItemGroup>

csharp/autobuilder/Semmle.Autobuild.CSharp/StandaloneBuildRule.cs

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,9 @@ internal class StandaloneBuildRule : IBuildRule<CSharpAutobuildOptions>
1010
{
1111
public BuildScript Analyse(IAutobuilder<CSharpAutobuildOptions> builder, bool auto)
1212
{
13-
return BuildScript.Create(_ => Semmle.Extraction.CSharp.Standalone.Program.Main([]));
13+
return builder.Options.Binlog is string binlog
14+
? BuildScript.Create(_ => Semmle.Extraction.CSharp.Driver.Main(["--binlog", binlog]))
15+
: BuildScript.Create(_ => Semmle.Extraction.CSharp.Standalone.Program.Main([]));
1416
}
1517
}
1618
}

csharp/codeql-extractor.yml

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -65,3 +65,9 @@ options:
6565
- progress+++
6666
type: string
6767
pattern: "^(off|errors|warnings|(info|progress)|(debug|progress\\+)|(trace|progress\\+\\+)|progress\\+\\+\\+)$"
68+
binlog:
69+
title: Binlog
70+
description: >
71+
[EXPERIMENTAL] The value is a path to the MsBuild binary log file that should be extracted.
72+
This option only works when `--build-mode none` is also specified.
73+
type: string

csharp/extractor/Semmle.Extraction.CSharp.Driver/BUILD.bazel

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,7 @@ codeql_csharp_binary(
88
srcs = glob([
99
"*.cs",
1010
]),
11-
visibility = ["//csharp:__pkg__"],
11+
visibility = ["//csharp:__subpackages__"],
1212
deps = [
1313
"//csharp/extractor/Semmle.Extraction.CSharp",
1414
],

csharp/extractor/Semmle.Extraction.CSharp/BUILD.bazel

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@ codeql_csharp_library(
1919
"//csharp/extractor/Semmle.Extraction",
2020
"//csharp/extractor/Semmle.Extraction.CSharp.Util",
2121
"//csharp/extractor/Semmle.Util",
22+
"@paket.main//basic.compilerlog.util",
2223
"@paket.main//microsoft.build",
2324
"@paket.main//microsoft.codeanalysis.csharp",
2425
],
Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
using Microsoft.CodeAnalysis.CSharp;
2+
using Semmle.Util;
3+
using Semmle.Util.Logging;
4+
5+
namespace Semmle.Extraction.CSharp
6+
{
7+
public class BinaryLogAnalyser : Analyser
8+
{
9+
public BinaryLogAnalyser(IProgressMonitor pm, ILogger logger, PathTransformer pathTransformer, IPathCache pathCache, bool addAssemblyTrapPrefix)
10+
: base(pm, logger, pathTransformer, pathCache, addAssemblyTrapPrefix)
11+
{
12+
}
13+
14+
public void Initialize(string cwd, string[] args, string outputPath, CSharpCompilation compilationIn, CommonOptions options)
15+
{
16+
compilation = compilationIn;
17+
ExtractionContext = new ExtractionContext(cwd, args, outputPath, [], Logger, PathTransformer, ExtractorMode.BinaryLog, options.QlTest);
18+
this.options = options;
19+
LogExtractorInfo();
20+
SetReferencePaths();
21+
}
22+
}
23+
}

csharp/extractor/Semmle.Extraction.CSharp/Extractor/Extractor.cs

Lines changed: 113 additions & 33 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@
88
using System.Text;
99
using System.Threading;
1010
using System.Threading.Tasks;
11+
using Basic.CompilerLog.Util;
1112
using Microsoft.CodeAnalysis;
1213
using Microsoft.CodeAnalysis.CSharp;
1314
using Microsoft.CodeAnalysis.Text;
@@ -102,55 +103,134 @@ public static ExitCode Run(string[] args)
102103

103104
try
104105
{
105-
if (options.ProjectsToLoad.Any())
106+
var canonicalPathCache = CanonicalPathCache.Create(logger, 1000);
107+
var pathTransformer = new PathTransformer(canonicalPathCache);
108+
109+
if (options.BinaryLogPath is string binlogPath)
106110
{
107-
AddSourceFilesFromProjects(options.ProjectsToLoad, options.CompilerArguments, logger);
111+
logger.LogInfo(" Running binary log analysis.");
112+
return RunBinaryLogAnalysis(analyzerStopwatch, options, binlogPath, logger, canonicalPathCache, pathTransformer);
108113
}
109-
110-
var compilerVersion = new CompilerVersion(options);
111-
if (compilerVersion.SkipExtraction)
114+
else
112115
{
113-
logger.LogWarning($" Unrecognized compiler '{compilerVersion.SpecifiedCompiler}' because {compilerVersion.SkipReason}");
114-
return ExitCode.Ok;
116+
logger.LogInfo(" Running tracing analysis.");
117+
return RunTracingAnalysis(analyzerStopwatch, options, logger, canonicalPathCache, pathTransformer);
115118
}
119+
}
120+
catch (Exception ex) // lgtm[cs/catch-of-all-exceptions]
121+
{
122+
logger.LogError($" Unhandled exception: {ex}");
123+
return ExitCode.Errors;
124+
}
125+
}
116126

117-
var workingDirectory = Directory.GetCurrentDirectory();
118-
var compilerArgs = options.CompilerArguments.ToArray();
127+
private static ExitCode RunBinaryLogAnalysis(Stopwatch stopwatch, Options options, string binlogPath, ILogger logger, CanonicalPathCache canonicalPathCache, PathTransformer pathTransformer)
128+
{
129+
logger.LogInfo($"Reading compiler calls from binary log {binlogPath}");
130+
try
131+
{
132+
using var fileStream = new FileStream(binlogPath, FileMode.Open, FileAccess.Read, FileShare.Read);
133+
using var reader = BinaryLogReader.Create(fileStream);
119134

120-
var canonicalPathCache = CanonicalPathCache.Create(logger, 1000);
121-
var pathTransformer = new PathTransformer(canonicalPathCache);
135+
// Filter out compiler calls that aren't interesting for examination
136+
static bool filter(CompilerCall compilerCall)
137+
{
138+
return compilerCall.IsCSharp &&
139+
compilerCall.Kind == CompilerCallKind.Regular;
140+
}
122141

123-
using var analyser = new TracingAnalyser(new LogProgressMonitor(logger), logger, pathTransformer, canonicalPathCache, options.AssemblySensitiveTrap);
142+
var allCompilationData = reader.ReadAllCompilationData(filter);
143+
var allFailed = true;
124144

125-
var compilerArguments = CSharpCommandLineParser.Default.Parse(
126-
compilerVersion.ArgsWithResponse,
127-
workingDirectory,
128-
compilerVersion.FrameworkPath,
129-
compilerVersion.AdditionalReferenceDirectories
130-
);
145+
logger.LogInfo($" Found {allCompilationData.Count} compilations in binary log");
131146

132-
if (compilerArguments is null)
147+
foreach (var compilationData in allCompilationData)
133148
{
134-
var sb = new StringBuilder();
135-
sb.Append(" Failed to parse command line: ").AppendList(" ", compilerArgs);
136-
logger.LogError(sb.ToString());
137-
++analyser.CompilationErrors;
138-
return ExitCode.Failed;
139-
}
149+
if (compilationData.GetCompilationAfterGenerators() is not CSharpCompilation compilation)
150+
{
151+
logger.LogError(" Compilation data is not C#");
152+
continue;
153+
}
140154

141-
if (!analyser.BeginInitialize(compilerVersion.ArgsWithResponse))
142-
{
143-
logger.LogInfo("Skipping extraction since files have already been extracted");
144-
return ExitCode.Ok;
155+
var compilerCall = compilationData.CompilerCall;
156+
var diagnosticName = compilerCall.GetDiagnosticName();
157+
logger.LogInfo($" Processing compilation {diagnosticName}");
158+
var compilerArgs = compilerCall.GetArguments();
159+
var args = reader.ReadCommandLineArguments(compilerCall);
160+
161+
using var analyser = new BinaryLogAnalyser(new LogProgressMonitor(logger), logger, pathTransformer, canonicalPathCache, options.AssemblySensitiveTrap);
162+
163+
var exit = Analyse(stopwatch, analyser, options,
164+
references => [() => compilation.References.ForEach(r => references.Add(r))],
165+
(analyser, syntaxTrees) => [() => syntaxTrees.AddRange(compilation.SyntaxTrees)],
166+
(syntaxTrees, references) => compilation,
167+
(compilation, options) => analyser.Initialize(compilerCall.ProjectDirectory, compilerArgs?.ToArray() ?? [], TracingAnalyser.GetOutputName(compilation, args), compilation, options),
168+
() => { });
169+
170+
switch (exit)
171+
{
172+
case ExitCode.Ok:
173+
allFailed &= false;
174+
logger.LogInfo($" Compilation {diagnosticName} succeeded");
175+
break;
176+
case ExitCode.Errors:
177+
allFailed &= false;
178+
logger.LogWarning($" Compilation {diagnosticName} had errors");
179+
break;
180+
case ExitCode.Failed:
181+
logger.LogWarning($" Compilation {diagnosticName} failed");
182+
return ExitCode.Failed;
183+
}
145184
}
185+
return allFailed ? ExitCode.Failed : ExitCode.Ok;
186+
}
187+
catch (IOException ex)
188+
{
189+
logger.LogError($"Failed to open binary log: {ex.Message}");
190+
return ExitCode.Failed;
191+
}
192+
}
193+
194+
private static ExitCode RunTracingAnalysis(Stopwatch analyzerStopwatch, Options options, ILogger logger, CanonicalPathCache canonicalPathCache, PathTransformer pathTransformer)
195+
{
196+
if (options.ProjectsToLoad.Any())
197+
{
198+
AddSourceFilesFromProjects(options.ProjectsToLoad, options.CompilerArguments, logger);
199+
}
146200

147-
return AnalyseTracing(workingDirectory, compilerArgs, analyser, compilerArguments, options, analyzerStopwatch);
201+
var compilerVersion = new CompilerVersion(options);
202+
if (compilerVersion.SkipExtraction)
203+
{
204+
logger.LogWarning($" Unrecognized compiler '{compilerVersion.SpecifiedCompiler}' because {compilerVersion.SkipReason}");
205+
return ExitCode.Ok;
148206
}
149-
catch (Exception ex) // lgtm[cs/catch-of-all-exceptions]
207+
208+
var workingDirectory = Directory.GetCurrentDirectory();
209+
var compilerArgs = options.CompilerArguments.ToArray();
210+
using var analyser = new TracingAnalyser(new LogProgressMonitor(logger), logger, pathTransformer, canonicalPathCache, options.AssemblySensitiveTrap);
211+
var compilerArguments = CSharpCommandLineParser.Default.Parse(
212+
compilerVersion.ArgsWithResponse,
213+
workingDirectory,
214+
compilerVersion.FrameworkPath,
215+
compilerVersion.AdditionalReferenceDirectories
216+
);
217+
218+
if (compilerArguments is null)
150219
{
151-
logger.LogError($" Unhandled exception: {ex}");
152-
return ExitCode.Errors;
220+
var sb = new StringBuilder();
221+
sb.Append(" Failed to parse command line: ").AppendList(" ", compilerArgs);
222+
logger.LogError(sb.ToString());
223+
++analyser.CompilationErrors;
224+
return ExitCode.Failed;
225+
}
226+
227+
if (!analyser.BeginInitialize(compilerVersion.ArgsWithResponse))
228+
{
229+
logger.LogInfo("Skipping extraction since files have already been extracted");
230+
return ExitCode.Ok;
153231
}
232+
233+
return AnalyseTracing(workingDirectory, compilerArgs, analyser, compilerArguments, options, analyzerStopwatch);
154234
}
155235

156236
private static void AddSourceFilesFromProjects(IEnumerable<string> projectsToLoad, IList<string> compilerArguments, ILogger logger)

csharp/extractor/Semmle.Extraction.CSharp/Extractor/Options.cs

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,11 @@ public sealed class Options : CommonOptions
3232
/// </summary>
3333
public bool AssemblySensitiveTrap { get; private set; } = false;
3434

35+
/// <summary>
36+
/// The path to the binary log file, or null if unspecified.
37+
/// </summary>
38+
public string? BinaryLogPath { get; set; }
39+
3540
public static Options CreateWithEnvironment(string[] arguments)
3641
{
3742
var options = new Options();
@@ -65,6 +70,9 @@ public override bool HandleOption(string key, string value)
6570
case "load-sources-from-project":
6671
ProjectsToLoad.Add(value);
6772
return true;
73+
case "binlog":
74+
BinaryLogPath = value;
75+
return true;
6876
default:
6977
return base.HandleOption(key, value);
7078
}

0 commit comments

Comments
 (0)