Skip to content

Commit 62382fc

Browse files
committed
C#: Fetch .NET in dependency manager instead of autobuilder
1 parent c6426d8 commit 62382fc

File tree

31 files changed

+306
-373
lines changed

31 files changed

+306
-373
lines changed

cpp/autobuilder/Semmle.Autobuild.Cpp/Program.cs

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,7 @@
11
using System;
2+
23
using Semmle.Autobuild.Shared;
4+
using Semmle.Util;
35

46
namespace Semmle.Autobuild.Cpp
57
{

csharp/CSharp.sln

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,10 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Semmle.Autobuild.CSharp.Tes
3434
EndProject
3535
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Semmle.Extraction.CSharp.DependencyStubGenerator", "extractor\Semmle.Extraction.CSharp.DependencyStubGenerator\Semmle.Extraction.CSharp.DependencyStubGenerator.csproj", "{0EDA21A3-ADD8-4C10-B494-58B12B526B76}"
3636
EndProject
37+
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Semmle.Autobuild.Cpp", "..\cpp\autobuilder\Semmle.Autobuild.Cpp\Semmle.Autobuild.Cpp.csproj", "{125C4FB7-34DA-442A-9095-3EA1514270CD}"
38+
EndProject
39+
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Semmle.Autobuild.Cpp.Tests", "..\cpp\autobuilder\Semmle.Autobuild.Cpp.Tests\Semmle.Autobuild.Cpp.Tests.csproj", "{72F369B7-0707-401A-802F-D526F272F9EE}"
40+
EndProject
3741
Global
3842
GlobalSection(SolutionConfigurationPlatforms) = preSolution
3943
Debug|Any CPU = Debug|Any CPU
@@ -102,6 +106,14 @@ Global
102106
{0EDA21A3-ADD8-4C10-B494-58B12B526B76}.Debug|Any CPU.Build.0 = Debug|Any CPU
103107
{0EDA21A3-ADD8-4C10-B494-58B12B526B76}.Release|Any CPU.ActiveCfg = Release|Any CPU
104108
{0EDA21A3-ADD8-4C10-B494-58B12B526B76}.Release|Any CPU.Build.0 = Release|Any CPU
109+
{125C4FB7-34DA-442A-9095-3EA1514270CD}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
110+
{125C4FB7-34DA-442A-9095-3EA1514270CD}.Debug|Any CPU.Build.0 = Debug|Any CPU
111+
{125C4FB7-34DA-442A-9095-3EA1514270CD}.Release|Any CPU.ActiveCfg = Release|Any CPU
112+
{125C4FB7-34DA-442A-9095-3EA1514270CD}.Release|Any CPU.Build.0 = Release|Any CPU
113+
{72F369B7-0707-401A-802F-D526F272F9EE}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
114+
{72F369B7-0707-401A-802F-D526F272F9EE}.Debug|Any CPU.Build.0 = Debug|Any CPU
115+
{72F369B7-0707-401A-802F-D526F272F9EE}.Release|Any CPU.ActiveCfg = Release|Any CPU
116+
{72F369B7-0707-401A-802F-D526F272F9EE}.Release|Any CPU.Build.0 = Release|Any CPU
105117
EndGlobalSection
106118
GlobalSection(SolutionProperties) = preSolution
107119
HideSolutionNode = FALSE

csharp/autobuilder/Semmle.Autobuild.CSharp.Tests/BuildScripts.cs

Lines changed: 4 additions & 33 deletions
Original file line numberDiff line numberDiff line change
@@ -558,8 +558,6 @@ public void TestVcVarsAllBatFiles()
558558
[Fact]
559559
public void TestLinuxBuildlessExtractionSuccess()
560560
{
561-
actions.RunProcess["dotnet --list-sdks"] = 0;
562-
actions.RunProcessOut["dotnet --list-sdks"] = "any version";
563561
actions.RunProcess[@"C:\codeql\csharp/tools/linux64/Semmle.Extraction.CSharp.Standalone"] = 0;
564562
actions.FileExists["csharp.log"] = true;
565563
actions.GetEnvironmentVariable["CODEQL_EXTRACTOR_CSHARP_TRAP_DIR"] = "";
@@ -569,14 +567,12 @@ public void TestLinuxBuildlessExtractionSuccess()
569567
actions.EnumerateDirectories[@"C:\Project"] = "";
570568

571569
var autobuilder = CreateAutoBuilder(false, buildless: "true");
572-
TestAutobuilderScript(autobuilder, 0, 2);
570+
TestAutobuilderScript(autobuilder, 0, 1);
573571
}
574572

575573
[Fact]
576574
public void TestLinuxBuildlessExtractionFailed()
577575
{
578-
actions.RunProcess["dotnet --list-sdks"] = 0;
579-
actions.RunProcessOut["dotnet --list-sdks"] = "any version";
580576
actions.RunProcess[@"C:\codeql\csharp/tools/linux64/Semmle.Extraction.CSharp.Standalone"] = 10;
581577
actions.FileExists["csharp.log"] = true;
582578
actions.GetEnvironmentVariable["CODEQL_EXTRACTOR_CSHARP_TRAP_DIR"] = "";
@@ -586,14 +582,12 @@ public void TestLinuxBuildlessExtractionFailed()
586582
actions.EnumerateDirectories[@"C:\Project"] = "";
587583

588584
var autobuilder = CreateAutoBuilder(false, buildless: "true");
589-
TestAutobuilderScript(autobuilder, 10, 2);
585+
TestAutobuilderScript(autobuilder, 10, 1);
590586
}
591587

592588
[Fact]
593589
public void TestLinuxBuildlessExtractionSolution()
594590
{
595-
actions.RunProcess["dotnet --list-sdks"] = 0;
596-
actions.RunProcessOut["dotnet --list-sdks"] = "any version";
597591
actions.RunProcess[@"C:\codeql\csharp/tools/linux64/Semmle.Extraction.CSharp.Standalone"] = 0;
598592
actions.FileExists["csharp.log"] = true;
599593
actions.GetEnvironmentVariable["CODEQL_EXTRACTOR_CSHARP_TRAP_DIR"] = "";
@@ -603,28 +597,7 @@ public void TestLinuxBuildlessExtractionSolution()
603597
actions.EnumerateDirectories[@"C:\Project"] = "";
604598

605599
var autobuilder = CreateAutoBuilder(false, buildless: "true");
606-
TestAutobuilderScript(autobuilder, 0, 2);
607-
}
608-
609-
[Fact]
610-
public void TestLinuxBuildlessExtractionNoDotnet()
611-
{
612-
actions.RunProcess["dotnet --list-sdks"] = 1;
613-
actions.RunProcessOut["dotnet --list-sdks"] = "";
614-
actions.RunProcess[@"chmod u+x scratch/.dotnet/dotnet-install.sh"] = 0;
615-
actions.RunProcess[@"scratch/.dotnet/dotnet-install.sh --channel release --version 8.0.101 --install-dir scratch/.dotnet"] = 0;
616-
actions.RunProcess[@"C:\codeql\csharp/tools/linux64/Semmle.Extraction.CSharp.Standalone --dotnet scratch/.dotnet"] = 0;
617-
actions.FileExists["csharp.log"] = true;
618-
actions.GetEnvironmentVariable["CODEQL_EXTRACTOR_CSHARP_TRAP_DIR"] = "";
619-
actions.GetEnvironmentVariable["CODEQL_EXTRACTOR_CSHARP_SOURCE_ARCHIVE_DIR"] = "";
620-
actions.GetEnvironmentVariable["CODEQL_EXTRACTOR_CSHARP_SCRATCH_DIR"] = "scratch";
621-
actions.EnumerateFiles[@"C:\Project"] = "foo.cs\ntest.sln";
622-
actions.EnumerateDirectories[@"C:\Project"] = "";
623-
actions.DownloadFiles.Add(("https://dot.net/v1/dotnet-install.sh", "scratch/.dotnet/dotnet-install.sh"));
624-
actions.CreateDirectories.Add(@"scratch/.dotnet");
625-
626-
var autobuilder = CreateAutoBuilder(false, buildless: "true");
627-
TestAutobuilderScript(autobuilder, 0, 4);
600+
TestAutobuilderScript(autobuilder, 0, 1);
628601
}
629602

630603
private void SkipVsWhere()
@@ -915,8 +888,6 @@ public void TestSkipNugetMsBuild()
915888
[Fact]
916889
public void TestSkipNugetBuildless()
917890
{
918-
actions.RunProcess["dotnet --list-sdks"] = 0;
919-
actions.RunProcessOut["dotnet --list-sdks"] = "any version";
920891
actions.RunProcess[@"C:\codeql\csharp/tools/linux64/Semmle.Extraction.CSharp.Standalone"] = 0;
921892
actions.FileExists["csharp.log"] = true;
922893
actions.GetEnvironmentVariable["CODEQL_EXTRACTOR_CSHARP_TRAP_DIR"] = "";
@@ -926,7 +897,7 @@ public void TestSkipNugetBuildless()
926897
actions.EnumerateDirectories[@"C:\Project"] = "";
927898

928899
var autobuilder = CreateAutoBuilder(false, buildless: "true");
929-
TestAutobuilderScript(autobuilder, 0, 2);
900+
TestAutobuilderScript(autobuilder, 0, 1);
930901
}
931902

932903

csharp/autobuilder/Semmle.Autobuild.CSharp/CSharpAutobuilder.cs

Lines changed: 3 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -50,11 +50,8 @@ public override BuildScript GetBuildScript()
5050
attempt = new BuildCommandRule(DotNetRule.WithDotNet).Analyse(this, false) & CheckExtractorRun(true);
5151
break;
5252
case CSharpBuildStrategy.Buildless:
53-
attempt = DotNetRule.WithDotNet(this, ensureDotNetAvailable: true, (dotNetPath, env) =>
54-
{
55-
// No need to check that the extractor has been executed in buildless mode
56-
return new StandaloneBuildRule(dotNetPath).Analyse(this, false);
57-
});
53+
// No need to check that the extractor has been executed in buildless mode
54+
attempt = new StandaloneBuildRule().Analyse(this, false);
5855
break;
5956
case CSharpBuildStrategy.MSBuild:
6057
attempt = new MsBuildRule().Analyse(this, false) & CheckExtractorRun(true);
@@ -84,7 +81,7 @@ public BuildScript CheckExtractorRun(bool warnOnFailure) =>
8481
return 0;
8582

8683
if (warnOnFailure)
87-
Log(Severity.Error, "No C# code detected during build.");
84+
Logger.LogError("No C# code detected during build.");
8885

8986
return 1;
9087
});

csharp/autobuilder/Semmle.Autobuild.CSharp/Constants.cs

Lines changed: 0 additions & 8 deletions
This file was deleted.

csharp/autobuilder/Semmle.Autobuild.CSharp/DotNetRule.cs

Lines changed: 8 additions & 155 deletions
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@
55
using Semmle.Util;
66
using Semmle.Util.Logging;
77
using Semmle.Autobuild.Shared;
8-
using Newtonsoft.Json.Linq;
8+
using Semmle.Extraction.CSharp.DependencyFetching;
99

1010
namespace Semmle.Autobuild.CSharp
1111
{
@@ -39,11 +39,11 @@ public BuildScript Analyse(IAutobuilder<CSharpAutobuildOptions> builder, bool au
3939

4040
if (notDotNetProject is not null)
4141
{
42-
builder.Log(Severity.Info, "Not using .NET Core because of incompatible project {0}", notDotNetProject);
42+
builder.Logger.Log(Severity.Info, "Not using .NET Core because of incompatible project {0}", notDotNetProject);
4343
return BuildScript.Failure;
4444
}
4545

46-
builder.Log(Severity.Info, "Attempting to build using .NET Core");
46+
builder.Logger.LogInfo("Attempting to build using .NET Core");
4747
}
4848

4949
return WithDotNet(builder, ensureDotNetAvailable: false, (dotNetPath, environment) =>
@@ -81,29 +81,22 @@ public BuildScript Analyse(IAutobuilder<CSharpAutobuildOptions> builder, bool au
8181
/// </summary>
8282
public static BuildScript WithDotNet(IAutobuilder<AutobuildOptionsShared> builder, bool ensureDotNetAvailable, Func<string?, IDictionary<string, string>?, BuildScript> f)
8383
{
84-
var installDir = builder.Actions.PathCombine(FileUtils.GetTemporaryWorkingDirectory(builder.Actions.GetEnvironmentVariable, builder.Options.Language.UpperCaseName, out var _), ".dotnet");
85-
var installScript = DownloadDotNet(builder, installDir, ensureDotNetAvailable);
86-
return BuildScript.Bind(installScript, installed =>
84+
var temp = FileUtils.GetTemporaryWorkingDirectory(builder.Actions.GetEnvironmentVariable, builder.Options.Language.UpperCaseName, out var shouldCleanUp);
85+
return DotNet.WithDotNet(builder.Actions, builder.Logger, builder.Paths.Select(x => x.Item1), temp, shouldCleanUp, ensureDotNetAvailable, builder.Options.DotNetVersion, installDir =>
8786
{
8887
var env = new Dictionary<string, string>
8988
{
9089
{ "DOTNET_SKIP_FIRST_TIME_EXPERIENCE", "true" },
9190
{ "MSBUILDDISABLENODEREUSE", "1" }
9291
};
93-
if (installed == 0)
92+
if (installDir is not null)
9493
{
95-
// The installation succeeded, so use the newly installed .NET Core
94+
// The installation succeeded, so use the newly installed .NET
9695
var path = builder.Actions.GetEnvironmentVariable("PATH");
9796
var delim = builder.Actions.IsWindows() ? ";" : ":";
98-
env.Add("DOTNET_MULTILEVEL_LOOKUP", "false"); // prevent look up of other .NET Core SDKs
97+
env.Add("DOTNET_MULTILEVEL_LOOKUP", "false"); // prevent look up of other .NET SDKs
9998
env.Add("PATH", installDir + delim + path);
10099
}
101-
else
102-
{
103-
// The .NET SDK was not installed, either because the installation failed or because it was already installed.
104-
installDir = null;
105-
}
106-
107100
return f(installDir, env);
108101
});
109102
}
@@ -119,146 +112,6 @@ public static BuildScript WithDotNet(IAutobuilder<AutobuildOptionsShared> builde
119112
public static BuildScript WithDotNet(IAutobuilder<AutobuildOptionsShared> builder, Func<IDictionary<string, string>?, BuildScript> f)
120113
=> WithDotNet(builder, ensureDotNetAvailable: false, (_, env) => f(env));
121114

122-
/// <summary>
123-
/// Returns a script for downloading relevant versions of the
124-
/// .NET Core SDK. The SDK(s) will be installed at <code>installDir</code>
125-
/// (provided that the script succeeds).
126-
/// </summary>
127-
private static BuildScript DownloadDotNet(IAutobuilder<AutobuildOptionsShared> builder, string installDir, bool ensureDotNetAvailable)
128-
{
129-
if (!string.IsNullOrEmpty(builder.Options.DotNetVersion))
130-
// Specific version supplied in configuration: always use that
131-
return DownloadDotNetVersion(builder, installDir, builder.Options.DotNetVersion);
132-
133-
// Download versions mentioned in `global.json` files
134-
// See https://docs.microsoft.com/en-us/dotnet/core/tools/global-json
135-
var installScript = BuildScript.Success;
136-
var validGlobalJson = false;
137-
foreach (var path in builder.Paths.Select(p => p.Item1).Where(p => p.EndsWith("global.json", StringComparison.Ordinal)))
138-
{
139-
string version;
140-
try
141-
{
142-
var o = JObject.Parse(File.ReadAllText(path));
143-
version = (string)(o?["sdk"]?["version"]!);
144-
}
145-
catch // lgtm[cs/catch-of-all-exceptions]
146-
{
147-
// not a valid global.json file
148-
continue;
149-
}
150-
151-
installScript &= DownloadDotNetVersion(builder, installDir, version);
152-
validGlobalJson = true;
153-
}
154-
155-
if (validGlobalJson)
156-
{
157-
return installScript;
158-
}
159-
160-
if (ensureDotNetAvailable)
161-
{
162-
return DownloadDotNetVersion(builder, installDir, Constants.LatestDotNetSdkVersion, needExactVersion: false);
163-
}
164-
165-
return BuildScript.Failure;
166-
}
167-
168-
/// <summary>
169-
/// Returns a script for downloading a specific .NET Core SDK version, if the
170-
/// version is not already installed.
171-
///
172-
/// See https://docs.microsoft.com/en-us/dotnet/core/tools/dotnet-install-script.
173-
/// </summary>
174-
private static BuildScript DownloadDotNetVersion(IAutobuilder<AutobuildOptionsShared> builder, string path, string version, bool needExactVersion = true)
175-
{
176-
return BuildScript.Bind(GetInstalledSdksScript(builder.Actions), (sdks, sdksRet) =>
177-
{
178-
if (needExactVersion && sdksRet == 0 && sdks.Count == 1 && sdks[0].StartsWith(version + " ", StringComparison.Ordinal))
179-
{
180-
// The requested SDK is already installed (and no other SDKs are installed), so
181-
// no need to reinstall
182-
return BuildScript.Failure;
183-
}
184-
else if (!needExactVersion && sdksRet == 0 && sdks.Count > 0)
185-
{
186-
// there's at least one SDK installed, so no need to reinstall
187-
return BuildScript.Failure;
188-
}
189-
else if (!needExactVersion && sdksRet != 0)
190-
{
191-
builder.Log(Severity.Info, "No .NET Core SDK found.");
192-
}
193-
194-
builder.Log(Severity.Info, "Attempting to download .NET Core {0}", version);
195-
196-
if (builder.Actions.IsWindows())
197-
{
198-
199-
var psCommand = $"[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12; &([scriptblock]::Create((Invoke-WebRequest -UseBasicParsing 'https://dot.net/v1/dotnet-install.ps1'))) -Version {version} -InstallDir {path}";
200-
201-
BuildScript GetInstall(string pwsh) =>
202-
new CommandBuilder(builder.Actions).
203-
RunCommand(pwsh).
204-
Argument("-NoProfile").
205-
Argument("-ExecutionPolicy").
206-
Argument("unrestricted").
207-
Argument("-Command").
208-
Argument("\"" + psCommand + "\"").
209-
Script;
210-
211-
return GetInstall("pwsh") | GetInstall("powershell");
212-
}
213-
else
214-
{
215-
var dotnetInstallPath = builder.Actions.PathCombine(FileUtils.GetTemporaryWorkingDirectory(
216-
builder.Actions.GetEnvironmentVariable,
217-
builder.Options.Language.UpperCaseName,
218-
out var shouldCleanUp), ".dotnet", "dotnet-install.sh");
219-
220-
var downloadDotNetInstallSh = BuildScript.DownloadFile(
221-
"https://dot.net/v1/dotnet-install.sh",
222-
dotnetInstallPath,
223-
e => builder.Log(Severity.Warning, $"Failed to download 'dotnet-install.sh': {e.Message}"));
224-
225-
var chmod = new CommandBuilder(builder.Actions).
226-
RunCommand("chmod").
227-
Argument("u+x").
228-
Argument(dotnetInstallPath);
229-
230-
var install = new CommandBuilder(builder.Actions).
231-
RunCommand(dotnetInstallPath).
232-
Argument("--channel").
233-
Argument("release").
234-
Argument("--version").
235-
Argument(version).
236-
Argument("--install-dir").
237-
Argument(path);
238-
239-
var buildScript = downloadDotNetInstallSh & chmod.Script & install.Script;
240-
241-
if (shouldCleanUp)
242-
{
243-
var removeScript = new CommandBuilder(builder.Actions).
244-
RunCommand("rm").
245-
Argument(dotnetInstallPath);
246-
buildScript &= removeScript.Script;
247-
}
248-
249-
return buildScript;
250-
}
251-
});
252-
}
253-
254-
private static BuildScript GetInstalledSdksScript(IBuildActions actions)
255-
{
256-
var listSdks = new CommandBuilder(actions, silent: true).
257-
RunCommand("dotnet").
258-
Argument("--list-sdks");
259-
return listSdks.Script;
260-
}
261-
262115
private static string DotNetCommand(IBuildActions actions, string? dotNetPath) =>
263116
dotNetPath is not null ? actions.PathCombine(dotNetPath, "dotnet") : "dotnet";
264117

csharp/autobuilder/Semmle.Autobuild.CSharp/Semmle.Autobuild.CSharp.csproj

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@
2020
<ItemGroup>
2121
<ProjectReference Include="..\..\extractor\Semmle.Util\Semmle.Util.csproj" />
2222
<ProjectReference Include="..\..\extractor\Semmle.Extraction.CSharp\Semmle.Extraction.CSharp.csproj" />
23+
<ProjectReference Include="..\..\extractor\Semmle.Extraction.CSharp.DependencyFetching\Semmle.Extraction.CSharp.DependencyFetching.csproj" />
2324
<ProjectReference Include="..\Semmle.Autobuild.Shared\Semmle.Autobuild.Shared.csproj" />
2425
</ItemGroup>
2526
</Project>

csharp/autobuilder/Semmle.Autobuild.CSharp/StandaloneBuildRule.cs

Lines changed: 0 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -8,13 +8,6 @@ namespace Semmle.Autobuild.CSharp
88
/// </summary>
99
internal class StandaloneBuildRule : IBuildRule<CSharpAutobuildOptions>
1010
{
11-
private readonly string? dotNetPath;
12-
13-
internal StandaloneBuildRule(string? dotNetPath)
14-
{
15-
this.dotNetPath = dotNetPath;
16-
}
17-
1811
public BuildScript Analyse(IAutobuilder<CSharpAutobuildOptions> builder, bool auto)
1912
{
2013
if (builder.CodeQLExtractorLangRoot is null
@@ -27,12 +20,6 @@ public BuildScript Analyse(IAutobuilder<CSharpAutobuildOptions> builder, bool au
2720
var cmd = new CommandBuilder(builder.Actions);
2821
cmd.RunCommand(standalone);
2922

30-
if (!string.IsNullOrEmpty(this.dotNetPath))
31-
{
32-
cmd.Argument("--dotnet");
33-
cmd.QuoteArgument(this.dotNetPath);
34-
}
35-
3623
return cmd.Script;
3724
}
3825
}

0 commit comments

Comments
 (0)