Merge pull request github#16747 from tamasvajk/buildless/binary-log-extractor-2

C#: Add binlog support to buildless with source generator support

Author: tamasvajk

csharp/autobuilder/Semmle.Autobuild.CSharp/BUILD.bazel

@@ -13,6 +13,7 @@ codeql_csharp_binary(
         "//csharp/autobuilder/Semmle.Autobuild.Shared",
         "//csharp/extractor/Semmle.Extraction.CSharp",
         "//csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching",
+        "//csharp/extractor/Semmle.Extraction.CSharp.Driver:bin/Semmle.Extraction.CSharp.Driver",
         "//csharp/extractor/Semmle.Extraction.CSharp.Standalone:bin/Semmle.Extraction.CSharp.Standalone",
         "//csharp/extractor/Semmle.Util",
         "@paket.main//microsoft.build",

csharp/autobuilder/Semmle.Autobuild.CSharp/CSharpAutobuilder.cs

@@ -13,8 +13,10 @@ public class CSharpAutobuildOptions : AutobuildOptionsShared
     {
         private const string buildModeEnvironmentVariable = "CODEQL_EXTRACTOR_CSHARP_BUILD_MODE";
         internal const string ExtractorOptionBuildless = "CODEQL_EXTRACTOR_CSHARP_OPTION_BUILDLESS";
+        internal const string ExtractorOptionBinlog = "CODEQL_EXTRACTOR_CSHARP_OPTION_BINLOG";
 
         public bool Buildless { get; }
+        public string? Binlog { get; }
 
         public override Language Language => Language.CSharp;
 
@@ -29,7 +31,7 @@ public CSharpAutobuildOptions(IBuildActions actions) : base(actions)
                 actions.GetEnvironmentVariable(ExtractorOptionBuildless).AsBool("buildless", false) ||
                 actions.GetEnvironmentVariable(buildModeEnvironmentVariable)?.ToLower() == "none";
 
-
+            Binlog = actions.GetEnvironmentVariable(ExtractorOptionBinlog);
         }
     }
 
@@ -114,6 +116,20 @@ private BuildScript AddBuildlessStartedDiagnostic()
                     markdownMessage: "C# was extracted with build-mode set to 'none'. This means that all C# source in the working directory will be scanned, with build tools, such as Nuget and Dotnet CLIs, only contributing information about external dependencies.",
                     severity: DiagnosticMessage.TspSeverity.Note
                 ));
+
+                // For the time being we are adding an additional message regarding the binlog usage. In the future, we might want to remove the buildless messages altogether when the binlog option is specified.
+                if (actions.GetEnvironmentVariable(CSharpAutobuildOptions.ExtractorOptionBinlog) is not null)
+                {
+                    AddDiagnostic(new DiagnosticMessage(
+                        Options.Language,
+                        "buildless/binlog",
+                        "C# was extracted with the experimental 'binlog' option",
+                        visibility: new DiagnosticMessage.TspVisibility(statusPage: true, cliSummaryTable: true, telemetry: true),
+                        markdownMessage: "C# was extracted with the experimental 'binlog' option.",
+                        severity: DiagnosticMessage.TspSeverity.Note
+                    ));
+                }
+
                 return 0;
             });
         }

csharp/autobuilder/Semmle.Autobuild.CSharp/Semmle.Autobuild.CSharp.csproj

@@ -6,6 +6,7 @@
     <ProjectReference Include="..\..\extractor\Semmle.Util\Semmle.Util.csproj" />
     <ProjectReference Include="..\..\extractor\Semmle.Extraction.CSharp\Semmle.Extraction.CSharp.csproj" />
     <ProjectReference Include="..\..\extractor\Semmle.Extraction.CSharp.Standalone\Semmle.Extraction.CSharp.Standalone.csproj" />
+    <ProjectReference Include="..\..\extractor\Semmle.Extraction.CSharp.Driver\Semmle.Extraction.CSharp.Driver.csproj" />
     <ProjectReference Include="..\..\extractor\Semmle.Extraction.CSharp.DependencyFetching\Semmle.Extraction.CSharp.DependencyFetching.csproj" />
     <ProjectReference Include="..\Semmle.Autobuild.Shared\Semmle.Autobuild.Shared.csproj" />
   </ItemGroup>

csharp/autobuilder/Semmle.Autobuild.CSharp/StandaloneBuildRule.cs

@@ -10,7 +10,9 @@ internal class StandaloneBuildRule : IBuildRule<CSharpAutobuildOptions>
     {
         public BuildScript Analyse(IAutobuilder<CSharpAutobuildOptions> builder, bool auto)
         {
-            return BuildScript.Create(_ => Semmle.Extraction.CSharp.Standalone.Program.Main([]));
+            return builder.Options.Binlog is string binlog
+                ? BuildScript.Create(_ => Semmle.Extraction.CSharp.Driver.Main(["--binlog", binlog]))
+                : BuildScript.Create(_ => Semmle.Extraction.CSharp.Standalone.Program.Main([]));
         }
     }
 }

csharp/codeql-extractor.yml

@@ -65,3 +65,9 @@ options:
             - progress+++
         type: string
         pattern: "^(off|errors|warnings|(info|progress)|(debug|progress\\+)|(trace|progress\\+\\+)|progress\\+\\+\\+)$"
+  binlog:
+    title: Binlog
+    description: >
+      [EXPERIMENTAL] The value is a path to the MsBuild binary log file that should be extracted.
+      This option only works when `--build-mode none` is also specified.
+    type: string

csharp/extractor/Semmle.Extraction.CSharp.Driver/BUILD.bazel

@@ -8,7 +8,7 @@ codeql_csharp_binary(
     srcs = glob([
         "*.cs",
     ]),
-    visibility = ["//csharp:__pkg__"],
+    visibility = ["//csharp:__subpackages__"],
     deps = [
         "//csharp/extractor/Semmle.Extraction.CSharp",
     ],

csharp/extractor/Semmle.Extraction.CSharp/BUILD.bazel

@@ -19,6 +19,7 @@ codeql_csharp_library(
         "//csharp/extractor/Semmle.Extraction",
         "//csharp/extractor/Semmle.Extraction.CSharp.Util",
         "//csharp/extractor/Semmle.Util",
+        "@paket.main//basic.compilerlog.util",
         "@paket.main//microsoft.build",
         "@paket.main//microsoft.codeanalysis.csharp",
     ],

csharp/extractor/Semmle.Extraction.CSharp/Entities/File.cs

@@ -34,7 +34,14 @@ public override void Populate(TextWriter trapFile)
                         lineCounts.Total++;
 
                     trapFile.numlines(this, lineCounts);
-                    Context.TrapWriter.Archive(originalPath, TransformedPath, text.Encoding ?? System.Text.Encoding.Default);
+                    if (BinaryLogExtractionContext.GetAdjustedPath(Context.ExtractionContext, originalPath) is not null)
+                    {
+                        Context.TrapWriter.ArchiveContent(rawText, TransformedPath);
+                    }
+                    else
+                    {
+                        Context.TrapWriter.Archive(originalPath, TransformedPath, text.Encoding ?? System.Text.Encoding.Default);
+                    }
                 }
             }
             else if (IsPossiblyTextFile())

csharp/extractor/Semmle.Extraction.CSharp/Extractor/Analyser.cs

@@ -185,7 +185,8 @@ private void DoExtractTree(SyntaxTree tree)
             {
                 var stopwatch = new Stopwatch();
                 stopwatch.Start();
-                var sourcePath = tree.FilePath;
+                var sourcePath = BinaryLogExtractionContext.GetAdjustedPath(ExtractionContext, tree.FilePath) ?? tree.FilePath;
+
                 var transformedSourcePath = PathTransformer.Transform(sourcePath);
 
                 var trapPath = transformedSourcePath.GetTrapPath(Logger, options.TrapCompression);

csharp/extractor/Semmle.Extraction.CSharp/Extractor/BinaryLogAnalyser.cs

@@ -0,0 +1,29 @@
+using System.Collections.Generic;
+using Microsoft.CodeAnalysis.CSharp;
+using Semmle.Util;
+using Semmle.Util.Logging;
+
+namespace Semmle.Extraction.CSharp
+{
+    public class BinaryLogAnalyser : Analyser
+    {
+        public BinaryLogAnalyser(IProgressMonitor pm, ILogger logger, PathTransformer pathTransformer, IPathCache pathCache, bool addAssemblyTrapPrefix)
+            : base(pm, logger, pathTransformer, pathCache, addAssemblyTrapPrefix)
+        {
+        }
+
+        public void Initialize(
+            string cwd, string[] args, string outputPath, CSharpCompilation compilation,
+            IEnumerable<Microsoft.CodeAnalysis.SyntaxTree> generatedSyntaxTrees,
+            string compilationIdentifier, CommonOptions options)
+        {
+            base.compilation = compilation;
+            ExtractionContext = new BinaryLogExtractionContext(
+                cwd, args, outputPath, generatedSyntaxTrees, compilationIdentifier,
+                Logger, PathTransformer, options.QlTest);
+            this.options = options;
+            LogExtractorInfo();
+            SetReferencePaths();
+        }
+    }
+}