Skip to content

Commit c2c7826

Browse files
tamasvajktamasvajk
committed
C#: Extract dependency restore telemetry data
1 parent 6fbbb82 commit c2c7826

File tree

13 files changed

+133
-24
lines changed

13 files changed

+133
-24
lines changed

‎csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DependencyManager.cs

Lines changed: 65 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -133,6 +133,17 @@ public DependencyManager(string srcDir, IDependencyOptions options, ILogger logg
133133
logger.LogInfo($"{conflictedReferences,align} resolved assembly conflicts");
134134
logger.LogInfo($"{dotnetFrameworkVersionVariantCount,align} restored .NET framework variants");
135135
logger.LogInfo($"Build analysis completed in {DateTime.Now - startTime}");
136+
137+
CompilationInfos.AddRange([
138+
("Source files on filesystem", nonGeneratedSources.Count.ToString()),
139+
("Source files generated", generatedSources.Count.ToString()),
140+
("Solution files on filesystem", allSolutions.Count.ToString()),
141+
("Project files on filesystem", allProjects.Count.ToString()),
142+
("Resolved references", usedReferences.Keys.Count.ToString()),
143+
("Unresolved references", unresolvedReferences.Count.ToString()),
144+
("Resolved assembly conflicts", conflictedReferences.ToString()),
145+
("Restored .NET framework variants", dotnetFrameworkVersionVariantCount.ToString()),
146+
]);
136147
}
137148

138149
private HashSet<string> AddFrameworkDlls(HashSet<string> dllPaths)
@@ -146,12 +157,18 @@ private HashSet<string> AddFrameworkDlls(HashSet<string> dllPaths)
146157
return frameworkLocations;
147158
}
148159

149-
private void RestoreNugetPackages(List<FileInfo> allNonBinaryFiles, IEnumerable<string> allProjects, IEnumerable<string> allSolutions, HashSet<string> dllPaths)
160+
private void RestoreNugetPackages(List<FileInfo> allNonBinaryFiles, List<string> allProjects, List<string> allSolutions, HashSet<string> dllPaths)
150161
{
151162
try
152163
{
153164
var nuget = new NugetPackages(sourceDir.FullName, legacyPackageDirectory, logger);
154-
nuget.InstallPackages();
165+
var count = nuget.InstallPackages();
166+
167+
if (nuget.PackageCount > 0)
168+
{
169+
CompilationInfos.Add(("packages.config files", nuget.PackageCount.ToString()));
170+
CompilationInfos.Add(("Successfully restored packages.config files", count.ToString()));
171+
}
155172

156173
var nugetPackageDlls = legacyPackageDirectory.DirInfo.GetFiles("*.dll", new EnumerationOptions { RecurseSubdirectories = true });
157174
var nugetPackageDllPaths = nugetPackageDlls.Select(f => f.FullName).ToHashSet();
@@ -629,6 +646,11 @@ private void ResolveConflicts(IEnumerable<string> frameworkPaths)
629646
/// </summary>
630647
public IEnumerable<string> UnresolvedReferences => unresolvedReferences.Select(r => r.Key);
631648

649+
/// <summary>
650+
/// List of `(key, value)` tuples, that are stored in the DB for telemetry purposes.
651+
/// </summary>
652+
public List<(string, string)> CompilationInfos { get; } = new List<(string, string)>();
653+
632654
/// <summary>
633655
/// Record that a particular reference couldn't be resolved.
634656
/// Note that this records at most one project file per missing reference.
@@ -697,17 +719,24 @@ private void AnalyseProject(FileInfo project)
697719
/// Returns a list of projects that are up to date with respect to restore.
698720
/// </summary>
699721
/// <param name="solutions">A list of paths to solution files.</param>
700-
private IEnumerable<string> RestoreSolutions(IEnumerable<string> solutions, out IEnumerable<string> assets)
722+
private IEnumerable<string> RestoreSolutions(List<string> solutions, out IEnumerable<string> assets)
701723
{
724+
var successCount = 0;
702725
var assetFiles = new List<string>();
703726
var projects = solutions.SelectMany(solution =>
704727
{
705728
logger.LogInfo($"Restoring solution {solution}...");
706729
var res = dotnet.Restore(new(solution, packageDirectory.DirInfo.FullName, ForceDotnetRefAssemblyFetching: true));
730+
if (res.Success)
731+
{
732+
successCount++;
733+
}
707734
assetFiles.AddRange(res.AssetsFilePaths);
708735
return res.RestoredProjects;
709-
});
736+
}).ToList();
710737
assets = assetFiles;
738+
CompilationInfos.Add(("Successfully restored solution files", successCount.ToString()));
739+
CompilationInfos.Add(("Restored projects through solution files", projects.Count.ToString()));
711740
return projects;
712741
}
713742

@@ -719,14 +748,24 @@ private IEnumerable<string> RestoreSolutions(IEnumerable<string> solutions, out
719748
/// <param name="projects">A list of paths to project files.</param>
720749
private void RestoreProjects(IEnumerable<string> projects, out IEnumerable<string> assets)
721750
{
751+
var successCount = 0;
722752
var assetFiles = new List<string>();
753+
var sync = new object();
723754
Parallel.ForEach(projects, new ParallelOptions { MaxDegreeOfParallelism = options.Threads }, project =>
724755
{
725756
logger.LogInfo($"Restoring project {project}...");
726757
var res = dotnet.Restore(new(project, packageDirectory.DirInfo.FullName, ForceDotnetRefAssemblyFetching: true));
727-
assetFiles.AddRange(res.AssetsFilePaths);
758+
lock (sync)
759+
{
760+
if (res.Success)
761+
{
762+
successCount++;
763+
}
764+
assetFiles.AddRange(res.AssetsFilePaths);
765+
}
728766
});
729767
assets = assetFiles;
768+
CompilationInfos.Add(("Successfully restored project files", successCount.ToString()));
730769
}
731770

732771
private void DownloadMissingPackages(List<FileInfo> allFiles, ISet<string> dllPaths)
@@ -767,6 +806,11 @@ private void DownloadMissingPackages(List<FileInfo> allFiles, ISet<string> dllPa
767806
logger.LogInfo($"Using nuget.config file {nugetConfig}.");
768807
}
769808

809+
CompilationInfos.Add(("Fallback nuget restore", notYetDownloadedPackages.Count.ToString()));
810+
811+
var successCount = 0;
812+
var sync = new object();
813+
770814
Parallel.ForEach(notYetDownloadedPackages, new ParallelOptions { MaxDegreeOfParallelism = options.Threads }, package =>
771815
{
772816
logger.LogInfo($"Restoring package {package}...");
@@ -798,9 +842,25 @@ private void DownloadMissingPackages(List<FileInfo> allFiles, ISet<string> dllPa
798842
{
799843
logger.LogInfo($"Failed to restore nuget package {package}");
800844
}
845+
else
846+
{
847+
lock (sync)
848+
{
849+
successCount++;
850+
}
851+
}
852+
}
853+
else
854+
{
855+
lock (sync)
856+
{
857+
successCount++;
858+
}
801859
}
802860
});
803861

862+
CompilationInfos.Add(("Successfully ran fallback nuget restore", successCount.ToString()));
863+
804864
dllPaths.Add(missingPackageDirectory.DirInfo.FullName);
805865
}
806866

‎csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/NugetPackages.cs

Lines changed: 14 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,8 @@ internal class NugetPackages
2222
/// </summary>
2323
private readonly FileInfo[] packageFiles;
2424

25+
public int PackageCount => packageFiles.Length;
26+
2527
/// <summary>
2628
/// The computed packages directory.
2729
/// This will be in the Temp location
@@ -105,7 +107,7 @@ private string DownloadNugetExe(string sourceDir)
105107
/// Restore all files in a specified package.
106108
/// </summary>
107109
/// <param name="package">The package file.</param>
108-
private void RestoreNugetPackage(string package)
110+
private bool RestoreNugetPackage(string package)
109111
{
110112
logger.LogInfo($"Restoring file {package}...");
111113

@@ -141,22 +143,31 @@ private void RestoreNugetPackage(string package)
141143
if (exitCode != 0)
142144
{
143145
logger.LogError($"Command {pi.FileName} {pi.Arguments} failed with exit code {exitCode}");
146+
return false;
144147
}
145148
else
146149
{
147150
logger.LogInfo($"Restored file {package}");
151+
return true;
148152
}
149153
}
150154

151155
/// <summary>
152156
/// Download the packages to the temp folder.
153157
/// </summary>
154-
public void InstallPackages()
158+
public int InstallPackages()
155159
{
160+
var success = 0;
156161
foreach (var package in packageFiles)
157162
{
158-
RestoreNugetPackage(package.FullName);
163+
var result = RestoreNugetPackage(package.FullName);
164+
if (result)
165+
{
166+
success++;
167+
}
159168
}
169+
170+
return success;
160171
}
161172
}
162173
}

‎csharp/extractor/Semmle.Extraction.CSharp.Standalone/Extractor.cs

Lines changed: 9 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -24,8 +24,7 @@ private static IEnumerable<Action> GetResolvedReferencesStandalone(IEnumerable<s
2424

2525
private static void AnalyseStandalone(
2626
StandaloneAnalyser analyser,
27-
IEnumerable<string> sources,
28-
IEnumerable<string> referencePaths,
27+
ExtractionInput extractionInput,
2928
CommonOptions options,
3029
IProgressMonitor progressMonitor,
3130
Stopwatch stopwatch)
@@ -35,12 +34,12 @@ private static void AnalyseStandalone(
3534
try
3635
{
3736
CSharp.Extractor.Analyse(stopwatch, analyser, options,
38-
references => GetResolvedReferencesStandalone(referencePaths, references),
39-
(analyser, syntaxTrees) => CSharp.Extractor.ReadSyntaxTrees(sources, analyser, null, null, syntaxTrees),
37+
references => GetResolvedReferencesStandalone(extractionInput.References, references),
38+
(analyser, syntaxTrees) => CSharp.Extractor.ReadSyntaxTrees(extractionInput.Sources, analyser, null, null, syntaxTrees),
4039
(syntaxTrees, references) => CSharpCompilation.Create(
4140
output.Name, syntaxTrees, references, new CSharpCompilationOptions(OutputKind.ConsoleApplication, allowUnsafe: true)
4241
),
43-
(compilation, options) => analyser.Initialize(output.FullName, compilation, options),
42+
(compilation, options) => analyser.Initialize(output.FullName, extractionInput.CompilationInfos, compilation, options),
4443
_ => { },
4544
() =>
4645
{
@@ -73,8 +72,7 @@ private static void AnalyseStandalone(
7372
}
7473

7574
private static void ExtractStandalone(
76-
IEnumerable<string> sources,
77-
IEnumerable<string> referencePaths,
75+
ExtractionInput extractionInput,
7876
IProgressMonitor pm,
7977
ILogger logger,
8078
CommonOptions options)
@@ -88,7 +86,7 @@ private static void ExtractStandalone(
8886
using var analyser = new StandaloneAnalyser(pm, logger, false, pathTransformer);
8987
try
9088
{
91-
AnalyseStandalone(analyser, sources, referencePaths, options, pm, stopwatch);
89+
AnalyseStandalone(analyser, extractionInput, options, pm, stopwatch);
9290
}
9391
catch (Exception ex) // lgtm[cs/catch-of-all-exceptions]
9492
{
@@ -131,6 +129,8 @@ public void MissingSummary(int missingTypes, int missingNamespaces)
131129
}
132130
}
133131

132+
public record ExtractionInput(IEnumerable<string> Sources, IEnumerable<string> References, IEnumerable<(string, string)> CompilationInfos);
133+
134134
public static ExitCode Run(Options options)
135135
{
136136
var stopwatch = new Stopwatch();
@@ -152,8 +152,7 @@ public static ExitCode Run(Options options)
152152
logger.Log(Severity.Info, "");
153153
logger.Log(Severity.Info, "Extracting...");
154154
ExtractStandalone(
155-
a.Extraction.Sources,
156-
a.References,
155+
new ExtractionInput(a.Extraction.Sources, a.References, a.CompilationInfos),
157156
new ExtractionProgress(logger),
158157
fileLogger,
159158
options);

‎csharp/extractor/Semmle.Extraction.CSharp.Standalone/Program.cs

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,7 @@ public Analysis(ILogger logger, Options options)
3030
References = dependencyManager.ReferenceFiles;
3131
Extraction = new Extraction(options.SrcDir);
3232
Extraction.Sources.AddRange(dependencyManager.AllSourceFiles);
33+
CompilationInfos = dependencyManager.CompilationInfos;
3334
}
3435

3536
public IEnumerable<string> References { get; }
@@ -39,6 +40,8 @@ public Analysis(ILogger logger, Options options)
3940
/// </summary>
4041
public Extraction Extraction { get; }
4142

43+
public IEnumerable<(string, string)> CompilationInfos { get; }
44+
4245
private readonly DependencyManager dependencyManager;
4346

4447
public void Dispose()

‎csharp/extractor/Semmle.Extraction.CSharp/Extractor/Analyser.cs

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@
66
using System.Threading.Tasks;
77
using Microsoft.CodeAnalysis;
88
using Microsoft.CodeAnalysis.CSharp;
9+
using Semmle.Util;
910
using Semmle.Util.Logging;
1011
using Semmle.Extraction.CSharp.Populators;
1112

@@ -20,7 +21,7 @@ public class Analyser : IDisposable
2021
protected CSharpCompilation? compilation;
2122
protected CommonOptions? options;
2223
private protected Entities.Compilation? compilationEntity;
23-
private IDisposable? compilationTrapFile;
24+
private TrapWriter? compilationTrapFile;
2425

2526
private readonly object progressMutex = new object();
2627

@@ -240,6 +241,8 @@ private void DoAnalyseCompilation()
240241
var cx = new Context(extractor, compilation.Clone(), trapWriter, new AssemblyScope(assembly, assemblyPath), addAssemblyTrapPrefix);
241242

242243
compilationEntity = Entities.Compilation.Create(cx);
244+
245+
extractor.CompilationInfos?.ForEach(ci => trapWriter.Writer.compilation_info(compilationEntity, ci.key, ci.value));
243246
}
244247
catch (Exception ex) // lgtm[cs/catch-of-all-exceptions]
245248
{

‎csharp/extractor/Semmle.Extraction.CSharp/Extractor/StandaloneAnalyser.cs

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -13,10 +13,10 @@ public StandaloneAnalyser(IProgressMonitor pm, ILogger logger, bool addAssemblyT
1313
{
1414
}
1515

16-
public void Initialize(string outputPath, CSharpCompilation compilationIn, CommonOptions options)
16+
public void Initialize(string outputPath, IEnumerable<(string, string)> compilationInfos, CSharpCompilation compilationIn, CommonOptions options)
1717
{
1818
compilation = compilationIn;
19-
extractor = new StandaloneExtractor(outputPath, Logger, PathTransformer, options);
19+
extractor = new StandaloneExtractor(outputPath, compilationInfos, Logger, PathTransformer, options);
2020
this.options = options;
2121
LogExtractorInfo(Extraction.Extractor.Version);
2222
SetReferencePaths();

‎csharp/extractor/Semmle.Extraction.CSharp/Tuples.cs

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -71,6 +71,9 @@ internal static void compilation_args(this TextWriter trapFile, Compilation comp
7171
internal static void compilation_expanded_args(this TextWriter trapFile, Compilation compilation, int index, string arg) =>
7272
trapFile.WriteTuple("compilation_expanded_args", compilation, index, arg);
7373

74+
internal static void compilation_info(this TextWriter trapFile, Compilation compilation, string infoKey, string infoValue) =>
75+
trapFile.WriteTuple("compilation_info", compilation, infoKey, infoValue);
76+
7477
internal static void compilation_compiling_files(this TextWriter trapFile, Compilation compilation, int index, Extraction.Entities.File file) =>
7578
trapFile.WriteTuple("compilation_compiling_files", compilation, index, file);
7679

‎csharp/extractor/Semmle.Extraction/Extractor/Extractor.cs

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,8 @@
11
using System.Collections.Generic;
22
using Semmle.Util.Logging;
33

4+
using CompilationInfo = (string key, string value);
5+
46
namespace Semmle.Extraction
57
{
68
/// <summary>
@@ -10,17 +12,19 @@ public abstract class Extractor
1012
{
1113
public abstract ExtractorMode Mode { get; }
1214
public string OutputPath { get; }
15+
public IEnumerable<CompilationInfo> CompilationInfos { get; }
1316

1417
/// <summary>
1518
/// Creates a new extractor instance for one compilation unit.
1619
/// </summary>
1720
/// <param name="logger">The object used for logging.</param>
1821
/// <param name="pathTransformer">The object used for path transformations.</param>
19-
protected Extractor(string outputPath, ILogger logger, PathTransformer pathTransformer)
22+
protected Extractor(string outputPath, IEnumerable<CompilationInfo> compilationInfos, ILogger logger, PathTransformer pathTransformer)
2023
{
2124
OutputPath = outputPath;
2225
Logger = logger;
2326
PathTransformer = pathTransformer;
27+
CompilationInfos = compilationInfos;
2428
}
2529

2630
// Limit the number of error messages in the log file

‎csharp/extractor/Semmle.Extraction/Extractor/StandaloneExtractor.cs

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,4 @@
1+
using System.Collections.Generic;
12
using Semmle.Util.Logging;
23

34
namespace Semmle.Extraction
@@ -11,7 +12,7 @@ public class StandaloneExtractor : Extractor
1112
/// </summary>
1213
/// <param name="logger">The object used for logging.</param>
1314
/// <param name="pathTransformer">The object used for path transformations.</param>
14-
public StandaloneExtractor(string outputPath, ILogger logger, PathTransformer pathTransformer, CommonOptions options) : base(outputPath, logger, pathTransformer)
15+
public StandaloneExtractor(string outputPath, IEnumerable<(string, string)> compilationInfos, ILogger logger, PathTransformer pathTransformer, CommonOptions options) : base(outputPath, compilationInfos, logger, pathTransformer)
1516
{
1617
Mode = ExtractorMode.Standalone;
1718
if (options.QlTest)

‎csharp/extractor/Semmle.Extraction/Extractor/TracingExtractor.cs

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,4 @@
1+
using System.Linq;
12
using Semmle.Util.Logging;
23

34
namespace Semmle.Extraction
@@ -12,7 +13,7 @@ public class TracingExtractor : Extractor
1213
/// <param name="outputPath">The name of the output DLL/EXE, or null if not specified (standalone extraction).</param>
1314
/// <param name="logger">The object used for logging.</param>
1415
/// <param name="pathTransformer">The object used for path transformations.</param>
15-
public TracingExtractor(string outputPath, ILogger logger, PathTransformer pathTransformer, CommonOptions options) : base(outputPath, logger, pathTransformer)
16+
public TracingExtractor(string outputPath, ILogger logger, PathTransformer pathTransformer, CommonOptions options) : base(outputPath, Enumerable.Empty<(string, string)>(), logger, pathTransformer)
1617
{
1718
Mode = ExtractorMode.None;
1819
if (options.QlTest)

0 commit comments

Comments
 (0)