C#: Reduce disabled nullability regions by splitting 'Extractor' and 'Analyser'

Author: tamasvajk

csharp/extractor/Semmle.Extraction.CIL/Analyser.cs

@@ -7,7 +7,7 @@ namespace Semmle.Extraction.CIL
 {
     public static class Analyser
     {
-        private static void ExtractCIL(Extractor extractor, TrapWriter trapWriter, bool extractPdbs)
+        private static void ExtractCIL(NonStandaloneExtractor extractor, TrapWriter trapWriter, bool extractPdbs)
         {
             using var cilContext = new Context(extractor, trapWriter, extractor.OutputPath, extractPdbs);
             cilContext.Populate(new Assembly(cilContext));
@@ -33,7 +33,7 @@ public static void ExtractCIL(Layout layout, string assemblyPath, ILogger logger
             {
                 var canonicalPathCache = CanonicalPathCache.Create(logger, 1000);
                 var pathTransformer = new PathTransformer(canonicalPathCache);
-                var extractor = new Extractor(assemblyPath, logger, pathTransformer);
+                var extractor = new NonStandaloneExtractor(assemblyPath, logger, pathTransformer);
                 var transformedAssemblyPath = pathTransformer.Transform(assemblyPath);
                 var project = layout.LookupProjectOrDefault(transformedAssemblyPath);
                 using var trapWriter = project.CreateTrapWriter(logger, transformedAssemblyPath.WithSuffix(".cil"), trapCompression, discardDuplicates: true);

csharp/extractor/Semmle.Extraction.CSharp/Analyser.cs

@@ -18,7 +18,7 @@ private Assembly(Context cx, Microsoft.CodeAnalysis.Location? init)
             if (init == null)
             {
                 // This is the output assembly
-                assemblyPath = cx.Extractor.OutputPath;
+                assemblyPath = ((NonStandaloneExtractor)cx.Extractor).OutputPath;
                 assembly = cx.Compilation.Assembly;
             }
             else

csharp/extractor/Semmle.Extraction.CSharp/Entities/Assembly.cs

@@ -0,0 +1,311 @@
+using System;
+using Microsoft.CodeAnalysis;
+using Microsoft.CodeAnalysis.CSharp;
+using System.IO;
+using System.Linq;
+using Semmle.Extraction.CSharp.Populators;
+using System.Collections.Generic;
+using System.Threading.Tasks;
+using System.Diagnostics;
+using Semmle.Util.Logging;
+
+namespace Semmle.Extraction.CSharp
+{
+    /// <summary>
+    /// Encapsulates a C# analysis task.
+    /// </summary>
+    public class Analyser : IDisposable
+    {
+        protected Extraction.Extractor? extractor;
+        protected CSharpCompilation? compilation;
+        protected Layout? layout;
+        protected CommonOptions? options;
+
+        private readonly object progressMutex = new object();
+
+        // The bulk of the extraction work, potentially executed in parallel.
+        protected readonly List<Action> extractionTasks = new List<Action>();
+        private int taskCount = 0;
+
+        private readonly Stopwatch stopWatch = new Stopwatch();
+
+        private readonly IProgressMonitor progressMonitor;
+
+        public ILogger Logger { get; }
+
+        protected readonly bool addAssemblyTrapPrefix;
+
+        public PathTransformer PathTransformer { get; }
+
+        protected Analyser(IProgressMonitor pm, ILogger logger, bool addAssemblyTrapPrefix, PathTransformer pathTransformer)
+        {
+            Logger = logger;
+            this.addAssemblyTrapPrefix = addAssemblyTrapPrefix;
+            Logger.Log(Severity.Info, "EXTRACTION STARTING at {0}", DateTime.Now);
+            stopWatch.Start();
+            progressMonitor = pm;
+            PathTransformer = pathTransformer;
+        }
+
+        /// <summary>
+        /// Perform an analysis on a source file/syntax tree.
+        /// </summary>
+        /// <param name="tree">Syntax tree to analyse.</param>
+        public void AnalyseTree(SyntaxTree tree)
+        {
+            extractionTasks.Add(() => DoExtractTree(tree));
+        }
+
+#nullable disable warnings
+
+        /// <summary>
+        ///     Enqueue all reference analysis tasks.
+        /// </summary>
+        public void AnalyseReferences()
+        {
+            foreach (var assembly in compilation.References.OfType<PortableExecutableReference>())
+            {
+                // CIL first - it takes longer.
+                if (options.CIL)
+                    extractionTasks.Add(() => DoExtractCIL(assembly));
+                extractionTasks.Add(() => DoAnalyseReferenceAssembly(assembly));
+            }
+        }
+
+        /// <summary>
+        ///     Constructs the map from assembly string to its filename.
+        ///
+        ///     Roslyn doesn't record the relationship between a filename and its assembly
+        ///     information, so we need to retrieve this information manually.
+        /// </summary>
+        protected void SetReferencePaths()
+        {
+            foreach (var reference in compilation.References.OfType<PortableExecutableReference>())
+            {
+                try
+                {
+                    var refPath = reference.FilePath!;
+
+                    /*  This method is significantly faster and more lightweight than using
+                     *  System.Reflection.Assembly.ReflectionOnlyLoadFrom. It is also allows
+                     *  loading the same assembly from different locations.
+                     */
+                    using var pereader = new System.Reflection.PortableExecutable.PEReader(new FileStream(refPath, FileMode.Open, FileAccess.Read, FileShare.Read));
+
+                    var metadata = pereader.GetMetadata();
+                    string assemblyIdentity;
+                    unsafe
+                    {
+                        var reader = new System.Reflection.Metadata.MetadataReader(metadata.Pointer, metadata.Length);
+                        var def = reader.GetAssemblyDefinition();
+                        assemblyIdentity = reader.GetString(def.Name) + " " + def.Version;
+                    }
+                    extractor.SetAssemblyFile(assemblyIdentity, refPath);
+
+                }
+                catch (Exception ex)  // lgtm[cs/catch-of-all-exceptions]
+                {
+                    extractor.Message(new Message("Exception reading reference file", reference.FilePath, null, ex.StackTrace));
+                }
+            }
+        }
+
+        /// <summary>
+        ///     Extract an assembly to a new trap file.
+        ///     If the trap file exists, skip extraction to avoid duplicating
+        ///     extraction within the snapshot.
+        /// </summary>
+        /// <param name="r">The assembly to extract.</param>
+        private void DoAnalyseReferenceAssembly(PortableExecutableReference r)
+        {
+            try
+            {
+                var stopwatch = new Stopwatch();
+                stopwatch.Start();
+
+                var assemblyPath = r.FilePath!;
+                var transformedAssemblyPath = PathTransformer.Transform(assemblyPath);
+                var projectLayout = layout.LookupProjectOrDefault(transformedAssemblyPath);
+                using var trapWriter = projectLayout.CreateTrapWriter(Logger, transformedAssemblyPath, options.TrapCompression, discardDuplicates: true);
+
+                var skipExtraction = options.Cache && File.Exists(trapWriter.TrapFile);
+
+                if (!skipExtraction)
+                {
+                    /* Note on parallel builds:
+                     *
+                     * The trap writer and source archiver both perform atomic moves
+                     * of the file to the final destination.
+                     *
+                     * If the same source file or trap file are generated concurrently
+                     * (by different parallel invocations of the extractor), then
+                     * last one wins.
+                     *
+                     * Specifically, if two assemblies are analysed concurrently in a build,
+                     * then there is a small amount of duplicated work but the output should
+                     * still be correct.
+                     */
+
+                    // compilation.Clone() reduces memory footprint by allowing the symbols
+                    // in c to be garbage collected.
+                    Compilation c = compilation.Clone();
+
+
+                    if (c.GetAssemblyOrModuleSymbol(r) is IAssemblySymbol assembly)
+                    {
+                        var cx = new Context(extractor, c, trapWriter, new AssemblyScope(assembly, assemblyPath), addAssemblyTrapPrefix);
+
+                        foreach (var module in assembly.Modules)
+                        {
+                            AnalyseNamespace(cx, module.GlobalNamespace);
+                        }
+
+                        Entities.Attribute.ExtractAttributes(cx, assembly, Entities.Assembly.Create(cx, assembly.GetSymbolLocation()));
+
+                        cx.PopulateAll();
+                    }
+                }
+
+                ReportProgress(assemblyPath, trapWriter.TrapFile, stopwatch.Elapsed, skipExtraction ? AnalysisAction.UpToDate : AnalysisAction.Extracted);
+            }
+            catch (Exception ex)  // lgtm[cs/catch-of-all-exceptions]
+            {
+                Logger.Log(Severity.Error, "  Unhandled exception analyzing {0}: {1}", r.FilePath, ex);
+            }
+        }
+
+        private void DoExtractCIL(PortableExecutableReference r)
+        {
+            var stopwatch = new Stopwatch();
+            stopwatch.Start();
+            CIL.Analyser.ExtractCIL(layout, r.FilePath!, Logger, !options.Cache, options.PDB, options.TrapCompression, out var trapFile, out var extracted);
+            stopwatch.Stop();
+            ReportProgress(r.FilePath, trapFile, stopwatch.Elapsed, extracted ? AnalysisAction.Extracted : AnalysisAction.UpToDate);
+        }
+
+        private void DoExtractTree(SyntaxTree tree)
+        {
+            try
+            {
+                var stopwatch = new Stopwatch();
+                stopwatch.Start();
+                var sourcePath = tree.FilePath;
+                var transformedSourcePath = PathTransformer.Transform(sourcePath);
+
+                var projectLayout = layout.LookupProjectOrNull(transformedSourcePath);
+                var excluded = projectLayout == null;
+                var trapPath = excluded ? "" : projectLayout!.GetTrapPath(Logger, transformedSourcePath, options.TrapCompression);
+                var upToDate = false;
+
+                if (!excluded)
+                {
+                    // compilation.Clone() is used to allow symbols to be garbage collected.
+                    using var trapWriter = projectLayout!.CreateTrapWriter(Logger, transformedSourcePath, options.TrapCompression, discardDuplicates: false);
+
+                    upToDate = options.Fast && FileIsUpToDate(sourcePath, trapWriter.TrapFile);
+
+                    if (!upToDate)
+                    {
+                        var cx = new Context(extractor, compilation.Clone(), trapWriter, new SourceScope(tree), addAssemblyTrapPrefix);
+                        // Ensure that the file itself is populated in case the source file is totally empty
+                        var root = tree.GetRoot();
+                        Entities.File.Create(cx, root.SyntaxTree.FilePath);
+
+                        var csNode = (CSharpSyntaxNode)root;
+                        csNode.Accept(new CompilationUnitVisitor(cx));
+                        csNode.Accept(new DirectiveVisitor(cx));
+                        cx.PopulateAll();
+                        CommentPopulator.ExtractCommentBlocks(cx, cx.CommentGenerator);
+                        cx.PopulateAll();
+                    }
+                }
+
+                ReportProgress(sourcePath, trapPath, stopwatch.Elapsed, excluded
+                    ? AnalysisAction.Excluded
+                    : upToDate
+                        ? AnalysisAction.UpToDate
+                        : AnalysisAction.Extracted);
+            }
+            catch (Exception ex)  // lgtm[cs/catch-of-all-exceptions]
+            {
+                extractor.Message(new Message($"Unhandled exception processing syntax tree. {ex.Message}", tree.FilePath, null, ex.StackTrace));
+            }
+        }
+
+#nullable restore warnings
+
+        private static bool FileIsUpToDate(string src, string dest)
+        {
+            return File.Exists(dest) &&
+                File.GetLastWriteTime(dest) >= File.GetLastWriteTime(src);
+        }
+
+        private void AnalyseNamespace(Context cx, INamespaceSymbol ns)
+        {
+            foreach (var memberNamespace in ns.GetNamespaceMembers())
+            {
+                AnalyseNamespace(cx, memberNamespace);
+            }
+
+            foreach (var memberType in ns.GetTypeMembers())
+            {
+                Entities.Type.Create(cx, memberType).ExtractRecursive();
+            }
+        }
+
+        private void ReportProgress(string src, string output, TimeSpan time, AnalysisAction action)
+        {
+            lock (progressMutex)
+                progressMonitor.Analysed(++taskCount, extractionTasks.Count, src, output, time, action);
+        }
+
+        /// <summary>
+        /// Run all extraction tasks.
+        /// </summary>
+        /// <param name="numberOfThreads">The number of threads to use.</param>
+        public void PerformExtraction(int numberOfThreads)
+        {
+            Parallel.Invoke(
+                new ParallelOptions { MaxDegreeOfParallelism = numberOfThreads },
+                extractionTasks.ToArray());
+        }
+
+        public virtual void Dispose()
+        {
+            stopWatch.Stop();
+            Logger.Log(Severity.Info, "  Peak working set = {0} MB", Process.GetCurrentProcess().PeakWorkingSet64 / (1024 * 1024));
+
+            if (TotalErrors > 0)
+                Logger.Log(Severity.Info, "EXTRACTION FAILED with {0} error{1} in {2}", TotalErrors, TotalErrors == 1 ? "" : "s", stopWatch.Elapsed);
+            else
+                Logger.Log(Severity.Info, "EXTRACTION SUCCEEDED in {0}", stopWatch.Elapsed);
+
+            Logger.Dispose();
+        }
+
+        /// <summary>
+        /// Number of errors encountered during extraction.
+        /// </summary>
+        private int ExtractorErrors => extractor?.Errors ?? 0;
+
+        /// <summary>
+        /// Number of errors encountered by the compiler.
+        /// </summary>
+        public int CompilationErrors { get; set; }
+
+        /// <summary>
+        /// Total number of errors reported.
+        /// </summary>
+        public int TotalErrors => CompilationErrors + ExtractorErrors;
+
+        /// <summary>
+        /// Logs information about the extractor.
+        /// </summary>
+        public void LogExtractorInfo(string extractorVersion)
+        {
+            Logger.Log(Severity.Info, "  Extractor: {0}", Environment.GetCommandLineArgs().First());
+            Logger.Log(Severity.Info, "  Extractor version: {0}", extractorVersion);
+            Logger.Log(Severity.Info, "  Current working directory: {0}", Directory.GetCurrentDirectory());
+        }
+    }
+}

csharp/extractor/Semmle.Extraction.CSharp/Extractor/Analyser.cs

@@ -0,0 +1,12 @@
+namespace Semmle.Extraction.CSharp
+{
+    /// <summary>
+    /// What action was performed when extracting a file.
+    /// </summary>
+    public enum AnalysisAction
+    {
+        Extracted,
+        UpToDate,
+        Excluded
+    }
+}