Merge branch 'main' into rb/rm-library-models-cust-docs

Author: turbo

csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DependencyManager.cs

@@ -5,6 +5,7 @@
 using System.Linq;
 using System.Security.Cryptography;
 using System.Text;
+using System.Text.RegularExpressions;
 using System.Threading.Tasks;
 using Semmle.Util;
 using Semmle.Util.Logging;
@@ -14,7 +15,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
     /// <summary>
     /// Main implementation of the build analysis.
     /// </summary>
-    public sealed class DependencyManager : IDisposable
+    public sealed partial class DependencyManager : IDisposable
     {
         private readonly AssemblyCache assemblyCache;
         private readonly ILogger logger;
@@ -783,13 +784,53 @@ private void RestoreProjects(IEnumerable<string> projects, out IEnumerable<strin
             CompilationInfos.Add(("Successfully restored project files", successCount.ToString()));
         }
 
-        private void DownloadMissingPackages(List<FileInfo> allFiles, ISet<string> dllPaths)
+        [GeneratedRegex(@"^(.+)\.(\d+\.\d+\.\d+(-(.+))?)$", RegexOptions.IgnoreCase | RegexOptions.Compiled | RegexOptions.Singleline)]
+        private static partial Regex LegacyNugetPackage();
+
+
+        private static IEnumerable<string> GetRestoredPackageDirectoryNames(DirectoryInfo root)
         {
-            var alreadyDownloadedPackages = Directory.GetDirectories(packageDirectory.DirInfo.FullName)
+            return Directory.GetDirectories(root.FullName)
                 .Select(d => Path.GetFileName(d).ToLowerInvariant());
-            var notYetDownloadedPackages = fileContent.AllPackages
-                .Except(alreadyDownloadedPackages)
-                .ToList();
+        }
+
+        private IEnumerable<string> GetRestoredLegacyPackageNames()
+        {
+            var oldPackageDirectories = GetRestoredPackageDirectoryNames(legacyPackageDirectory.DirInfo);
+            foreach (var oldPackageDirectory in oldPackageDirectories)
+            {
+                // nuget install restores packages to 'packagename.version' folders (dotnet restore to 'packagename/version' folders)
+                // typical folder names look like:
+                //   newtonsoft.json.13.0.3
+                // there are more complex ones too, such as:
+                //   runtime.tizen.4.0.0-armel.Microsoft.NETCore.DotNetHostResolver.2.0.0-preview2-25407-01
+
+                var match = LegacyNugetPackage().Match(oldPackageDirectory);
+                if (!match.Success)
+                {
+                    logger.LogWarning($"Package directory '{oldPackageDirectory}' doesn't match the expected pattern.");
+                    continue;
+                }
+
+                yield return match.Groups[1].Value.ToLowerInvariant();
+            }
+        }
+
+        private void DownloadMissingPackages(List<FileInfo> allFiles, ISet<string> dllPaths)
+        {
+            var alreadyDownloadedPackages = GetRestoredPackageDirectoryNames(packageDirectory.DirInfo);
+            var alreadyDownloadedLegacyPackages = GetRestoredLegacyPackageNames();
+
+            var notYetDownloadedPackages = new HashSet<string>(fileContent.AllPackages);
+            foreach (var alreadyDownloadedPackage in alreadyDownloadedPackages)
+            {
+                notYetDownloadedPackages.Remove(alreadyDownloadedPackage);
+            }
+            foreach (var alreadyDownloadedLegacyPackage in alreadyDownloadedLegacyPackages)
+            {
+                notYetDownloadedPackages.Remove(alreadyDownloadedLegacyPackage);
+            }
+
             if (notYetDownloadedPackages.Count == 0)
             {
                 return;

csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/FileContent.cs

@@ -157,23 +157,35 @@ private static bool IsGroupMatch(ReadOnlySpan<char> line, Regex regex, string gr
             return false;
         }
 
+        private void AddPackageReference(ReadOnlySpan<char> line, string groupName, Func<Regex> regex)
+        {
+            foreach (var valueMatch in regex().EnumerateMatches(line))
+            {
+                // We can't get the group from the ValueMatch, so doing it manually:
+                var packageName = GetGroup(line, valueMatch, groupName).ToLowerInvariant();
+                if (!string.IsNullOrEmpty(packageName))
+                {
+                    allPackages.Add(packageName);
+                }
+            }
+        }
+
         private void DoInitialize()
         {
             foreach (var file in files)
             {
                 try
                 {
+                    var isPackagesConfig = file.EndsWith("packages.config", StringComparison.OrdinalIgnoreCase);
+
                     foreach (ReadOnlySpan<char> line in unsafeFileReader.ReadLines(file))
                     {
                         // Find all the packages.
-                        foreach (var valueMatch in PackageReference().EnumerateMatches(line))
+                        AddPackageReference(line, "Include", PackageReference);
+
+                        if (isPackagesConfig)
                         {
-                            // We can't get the group from the ValueMatch, so doing it manually:
-                            var packageName = GetGroup(line, valueMatch, "Include").ToLowerInvariant();
-                            if (!string.IsNullOrEmpty(packageName))
-                            {
-                                allPackages.Add(packageName);
-                            }
+                            AddPackageReference(line, "id", LegacyPackageReference);
                         }
 
                         // Determine if ASP.NET is used.
@@ -223,6 +235,9 @@ private void DoInitialize()
         [GeneratedRegex("(?<!<!--.*)<PackageReference.*\\sInclude=\"(.*?)\".*/?>", RegexOptions.IgnoreCase | RegexOptions.Compiled | RegexOptions.Singleline)]
         private static partial Regex PackageReference();
 
+        [GeneratedRegex("(?<!<!--.*)<package.*\\sid=\"(.*?)\".*/?>", RegexOptions.IgnoreCase | RegexOptions.Compiled | RegexOptions.Singleline)]
+        private static partial Regex LegacyPackageReference();
+
         [GeneratedRegex("(?<!<!--.*)<FrameworkReference.*\\sInclude=\"(.*?)\".*/?>", RegexOptions.IgnoreCase | RegexOptions.Compiled | RegexOptions.Singleline)]
         private static partial Regex FrameworkReference();
 

csharp/extractor/Semmle.Extraction.CSharp/Entities/Constructor.cs

@@ -1,3 +1,5 @@
+using System;
+using System.Collections.Generic;
 using System.Diagnostics.CodeAnalysis;
 using System.IO;
 using System.Linq;
@@ -10,8 +12,16 @@ namespace Semmle.Extraction.CSharp.Entities
 {
     internal class Constructor : Method
     {
+        private readonly List<SyntaxNode> declaringReferenceSyntax;
+
         private Constructor(Context cx, IMethodSymbol init)
-            : base(cx, init) { }
+            : base(cx, init)
+        {
+            declaringReferenceSyntax =
+                Symbol.DeclaringSyntaxReferences
+                    .Select(r => r.GetSyntax())
+                    .ToList();
+        }
 
         public override void Populate(TextWriter trapFile)
         {
@@ -22,6 +32,12 @@ public override void Populate(TextWriter trapFile)
             trapFile.constructors(this, Symbol.ContainingType.Name, ContainingType, (Constructor)OriginalDefinition);
             trapFile.constructor_location(this, Location);
 
+            if (IsPrimary)
+            {
+                // Create a synthetic empty body for primary constructors.
+                Statements.SyntheticEmptyBlock.Create(Context, this, 0, Location);
+            }
+
             if (Symbol.IsImplicitlyDeclared)
             {
                 var lineCounts = new LineCounts() { Total = 2, Code = 1, Comment = 0 };
@@ -33,68 +49,79 @@ public override void Populate(TextWriter trapFile)
         protected override void ExtractInitializers(TextWriter trapFile)
         {
             // Do not extract initializers for constructed types.
-            if (!IsSourceDeclaration)
+            // Only extract initializers for constructors with a body and primary constructors.
+            if (Block is null && ExpressionBody is null && !IsPrimary ||
+                !IsSourceDeclaration)
+            {
                 return;
+            }
 
-            var syntax = Syntax;
-            var initializer = syntax?.Initializer;
-
-            if (initializer is null)
+            if (OrdinaryConstructorSyntax?.Initializer is ConstructorInitializerSyntax initializer)
             {
-                if (Symbol.MethodKind is MethodKind.Constructor)
+                ITypeSymbol initializerType;
+                var initializerInfo = Context.GetSymbolInfo(initializer);
+
+                switch (initializer.Kind())
                 {
-                    var baseType = Symbol.ContainingType.BaseType;
-                    if (baseType is null)
-                    {
-                        if (Symbol.ContainingType.SpecialType != SpecialType.System_Object)
-                        {
-                            Context.ModelError(Symbol, "Unable to resolve base type in implicit constructor initializer");
-                        }
+                    case SyntaxKind.BaseConstructorInitializer:
+                        initializerType = Symbol.ContainingType.BaseType!;
+                        break;
+                    case SyntaxKind.ThisConstructorInitializer:
+                        initializerType = Symbol.ContainingType;
+                        break;
+                    default:
+                        Context.ModelError(initializer, "Unknown initializer");
                         return;
-                    }
+                }
 
-                    var baseConstructor = baseType.InstanceConstructors.FirstOrDefault(c => c.Arity is 0);
+                ExtractSourceInitializer(trapFile, initializerType, (IMethodSymbol?)initializerInfo.Symbol, initializer.ArgumentList, initializer.ThisOrBaseKeyword.GetLocation());
+            }
+            else if (PrimaryBase is PrimaryConstructorBaseTypeSyntax primaryInitializer)
+            {
+                var primaryInfo = Context.GetSymbolInfo(primaryInitializer);
+                var primarySymbol = primaryInfo.Symbol;
 
-                    if (baseConstructor is null)
+                ExtractSourceInitializer(trapFile, primarySymbol?.ContainingType, (IMethodSymbol?)primarySymbol, primaryInitializer.ArgumentList, primaryInitializer.GetLocation());
+            }
+            else if (Symbol.MethodKind is MethodKind.Constructor)
+            {
+                var baseType = Symbol.ContainingType.BaseType;
+                if (baseType is null)
+                {
+                    if (Symbol.ContainingType.SpecialType != SpecialType.System_Object)
                     {
-                        Context.ModelError(Symbol, "Unable to resolve implicit constructor initializer call");
-                        return;
+                        Context.ModelError(Symbol, "Unable to resolve base type in implicit constructor initializer");
                     }
-
-                    var baseConstructorTarget = Create(Context, baseConstructor);
-                    var info = new ExpressionInfo(Context,
-                        AnnotatedTypeSymbol.CreateNotAnnotated(baseType),
-                        Location,
-                        Kinds.ExprKind.CONSTRUCTOR_INIT,
-                        this,
-                        -1,
-                        isCompilerGenerated: true,
-                        null);
-
-                    trapFile.expr_call(new Expression(info), baseConstructorTarget);
+                    return;
                 }
-                return;
-            }
 
-            ITypeSymbol initializerType;
-            var symbolInfo = Context.GetSymbolInfo(initializer);
+                var baseConstructor = baseType.InstanceConstructors.FirstOrDefault(c => c.Arity is 0);
 
-            switch (initializer.Kind())
-            {
-                case SyntaxKind.BaseConstructorInitializer:
-                    initializerType = Symbol.ContainingType.BaseType!;
-                    break;
-                case SyntaxKind.ThisConstructorInitializer:
-                    initializerType = Symbol.ContainingType;
-                    break;
-                default:
-                    Context.ModelError(initializer, "Unknown initializer");
+                if (baseConstructor is null)
+                {
+                    Context.ModelError(Symbol, "Unable to resolve implicit constructor initializer call");
                     return;
+                }
+
+                var baseConstructorTarget = Create(Context, baseConstructor);
+                var info = new ExpressionInfo(Context,
+                    AnnotatedTypeSymbol.CreateNotAnnotated(baseType),
+                    Location,
+                    Kinds.ExprKind.CONSTRUCTOR_INIT,
+                    this,
+                    -1,
+                    isCompilerGenerated: true,
+                    null);
+
+                trapFile.expr_call(new Expression(info), baseConstructorTarget);
             }
+        }
 
+        private void ExtractSourceInitializer(TextWriter trapFile, ITypeSymbol? type, IMethodSymbol? symbol, ArgumentListSyntax arguments, Location location)
+        {
             var initInfo = new ExpressionInfo(Context,
-                AnnotatedTypeSymbol.CreateNotAnnotated(initializerType),
-                Context.CreateLocation(initializer.ThisOrBaseKeyword.GetLocation()),
+                AnnotatedTypeSymbol.CreateNotAnnotated(type),
+                Context.CreateLocation(location),
                 Kinds.ExprKind.CONSTRUCTOR_INIT,
                 this,
                 -1,
@@ -103,7 +130,7 @@ protected override void ExtractInitializers(TextWriter trapFile)
 
             var init = new Expression(initInfo);
 
-            var target = Constructor.Create(Context, (IMethodSymbol?)symbolInfo.Symbol);
+            var target = Constructor.Create(Context, symbol);
             if (target is null)
             {
                 Context.ModelError(Symbol, "Unable to resolve call");
@@ -112,19 +139,27 @@ protected override void ExtractInitializers(TextWriter trapFile)
 
             trapFile.expr_call(init, target);
 
-            init.PopulateArguments(trapFile, initializer.ArgumentList, 0);
+            init.PopulateArguments(trapFile, arguments, 0);
         }
 
-        private ConstructorDeclarationSyntax? Syntax
-        {
-            get
-            {
-                return Symbol.DeclaringSyntaxReferences
-                    .Select(r => r.GetSyntax())
-                    .OfType<ConstructorDeclarationSyntax>()
-                    .FirstOrDefault();
-            }
-        }
+        private ConstructorDeclarationSyntax? OrdinaryConstructorSyntax =>
+            declaringReferenceSyntax
+                .OfType<ConstructorDeclarationSyntax>()
+                .FirstOrDefault();
+
+        private TypeDeclarationSyntax? PrimaryConstructorSyntax =>
+            declaringReferenceSyntax
+                    .OfType<TypeDeclarationSyntax>()
+                    .FirstOrDefault(t => t is ClassDeclarationSyntax or StructDeclarationSyntax or RecordDeclarationSyntax);
+
+        private PrimaryConstructorBaseTypeSyntax? PrimaryBase =>
+            PrimaryConstructorSyntax?
+                .BaseList?
+                .Types
+                .OfType<PrimaryConstructorBaseTypeSyntax>()
+                .FirstOrDefault();
+
+        private bool IsPrimary => PrimaryConstructorSyntax is not null;
 
         [return: NotNullIfNotNull(nameof(constructor))]
         public static new Constructor? Create(Context cx, IMethodSymbol? constructor)
@@ -160,19 +195,20 @@ public override void WriteId(EscapingTextWriter trapFile)
             trapFile.Write(";constructor");
         }
 
-        private ConstructorDeclarationSyntax? GetSyntax() =>
-            Symbol.DeclaringSyntaxReferences.Select(r => r.GetSyntax()).OfType<ConstructorDeclarationSyntax>().FirstOrDefault();
-
         public override Microsoft.CodeAnalysis.Location? FullLocation => ReportingLocation;
 
         public override Microsoft.CodeAnalysis.Location? ReportingLocation
         {
             get
             {
-                var syn = GetSyntax();
-                if (syn is not null)
+                if (OrdinaryConstructorSyntax is not null)
+                {
+                    return OrdinaryConstructorSyntax.Identifier.GetLocation();
+                }
+
+                if (PrimaryConstructorSyntax is not null)
                 {
-                    return syn.Identifier.GetLocation();
+                    return PrimaryConstructorSyntax.Identifier.GetLocation();
                 }
 
                 if (Symbol.IsImplicitlyDeclared)

csharp/extractor/Semmle.Extraction.CSharp/Entities/Method.cs

@@ -54,12 +54,13 @@ protected virtual void PopulateMethodBody(TextWriter trapFile)
             var block = Block;
             var expr = ExpressionBody;
 
+            Context.PopulateLater(() => ExtractInitializers(trapFile));
+
             if (block is not null || expr is not null)
             {
                 Context.PopulateLater(
                    () =>
                    {
-                       ExtractInitializers(trapFile);
                        if (block is not null)
                            Statements.Block.Create(Context, block, this, 0);
                        else