Skip to content

Commit 5291fc2

Browse files
authored
Merge pull request #761 from jonsequitur/fix-ymls-9
add more ymls
2 parents 3a0e536 + db4fe44 commit 5291fc2

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

48 files changed

+3095
-639
lines changed

eng/common/CheckSymbols.ps1

Lines changed: 27 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -5,11 +5,12 @@ param(
55
)
66

77
Add-Type -AssemblyName System.IO.Compression.FileSystem
8+
. $PSScriptRoot\pipeline-logging-functions.ps1
89

910
function FirstMatchingSymbolDescriptionOrDefault {
1011
param(
1112
[string] $FullPath, # Full path to the module that has to be checked
12-
[string] $TargetServerParam, # Parameter to pass to `Symbol Tool` indicating the server to lookup for symbols
13+
[string] $TargetServerParameter, # Parameter to pass to `Symbol Tool` indicating the server to lookup for symbols
1314
[string] $SymbolsPath
1415
)
1516

@@ -21,36 +22,36 @@ function FirstMatchingSymbolDescriptionOrDefault {
2122
# checking and which type of file was uploaded.
2223

2324
# The file itself is returned
24-
$SymbolPath = $SymbolsPath + "\" + $FileName
25+
$SymbolPath = $SymbolsPath + '\' + $FileName
2526

2627
# PDB file for the module
27-
$PdbPath = $SymbolPath.Replace($Extension, ".pdb")
28+
$PdbPath = $SymbolPath.Replace($Extension, '.pdb')
2829

2930
# PDB file for R2R module (created by crossgen)
30-
$NGenPdb = $SymbolPath.Replace($Extension, ".ni.pdb")
31+
$NGenPdb = $SymbolPath.Replace($Extension, '.ni.pdb')
3132

3233
# DBG file for a .so library
33-
$SODbg = $SymbolPath.Replace($Extension, ".so.dbg")
34+
$SODbg = $SymbolPath.Replace($Extension, '.so.dbg')
3435

3536
# DWARF file for a .dylib
36-
$DylibDwarf = $SymbolPath.Replace($Extension, ".dylib.dwarf")
37+
$DylibDwarf = $SymbolPath.Replace($Extension, '.dylib.dwarf')
3738

38-
.\dotnet-symbol.exe --symbols --modules --windows-pdbs $TargetServerParam $FullPath -o $SymbolsPath | Out-Null
39+
.\dotnet-symbol.exe --symbols --modules --windows-pdbs $TargetServerParameter $FullPath -o $SymbolsPath | Out-Null
3940

4041
if (Test-Path $PdbPath) {
41-
return "PDB"
42+
return 'PDB'
4243
}
4344
elseif (Test-Path $NGenPdb) {
44-
return "NGen PDB"
45+
return 'NGen PDB'
4546
}
4647
elseif (Test-Path $SODbg) {
47-
return "DBG for SO"
48+
return 'DBG for SO'
4849
}
4950
elseif (Test-Path $DylibDwarf) {
50-
return "Dwarf for Dylib"
51+
return 'Dwarf for Dylib'
5152
}
5253
elseif (Test-Path $SymbolPath) {
53-
return "Module"
54+
return 'Module'
5455
}
5556
else {
5657
return $null
@@ -68,15 +69,15 @@ function CountMissingSymbols {
6869
}
6970

7071
# Extensions for which we'll look for symbols
71-
$RelevantExtensions = @(".dll", ".exe", ".so", ".dylib")
72+
$RelevantExtensions = @('.dll', '.exe', '.so', '.dylib')
7273

7374
# How many files are missing symbol information
7475
$MissingSymbols = 0
7576

7677
$PackageId = [System.IO.Path]::GetFileNameWithoutExtension($PackagePath)
7778
$PackageGuid = New-Guid
7879
$ExtractPath = Join-Path -Path $ExtractPath -ChildPath $PackageGuid
79-
$SymbolsPath = Join-Path -Path $ExtractPath -ChildPath "Symbols"
80+
$SymbolsPath = Join-Path -Path $ExtractPath -ChildPath 'Symbols'
8081

8182
[System.IO.Compression.ZipFile]::ExtractToDirectory($PackagePath, $ExtractPath)
8283

@@ -86,31 +87,31 @@ function CountMissingSymbols {
8687
Get-ChildItem -Recurse $ExtractPath |
8788
Where-Object {$RelevantExtensions -contains $_.Extension} |
8889
ForEach-Object {
89-
if ($_.FullName -Match "\\ref\\") {
90+
if ($_.FullName -Match '\\ref\\') {
9091
Write-Host "`t Ignoring reference assembly file" $_.FullName
9192
return
9293
}
9394

94-
$SymbolsOnMSDL = FirstMatchingSymbolDescriptionOrDefault $_.FullName "--microsoft-symbol-server" $SymbolsPath
95-
$SymbolsOnSymWeb = FirstMatchingSymbolDescriptionOrDefault $_.FullName "--internal-server" $SymbolsPath
95+
$SymbolsOnMSDL = FirstMatchingSymbolDescriptionOrDefault -FullPath $_.FullName -TargetServerParameter '--microsoft-symbol-server' -SymbolsPath $SymbolsPath
96+
$SymbolsOnSymWeb = FirstMatchingSymbolDescriptionOrDefault -FullPath $_.FullName -TargetServerParameter '--internal-server' -SymbolsPath $SymbolsPath
9697

9798
Write-Host -NoNewLine "`t Checking file" $_.FullName "... "
9899

99100
if ($SymbolsOnMSDL -ne $null -and $SymbolsOnSymWeb -ne $null) {
100-
Write-Host "Symbols found on MSDL (" $SymbolsOnMSDL ") and SymWeb (" $SymbolsOnSymWeb ")"
101+
Write-Host "Symbols found on MSDL (${$SymbolsOnMSDL}) and SymWeb (${$SymbolsOnSymWeb})"
101102
}
102103
else {
103104
$MissingSymbols++
104105

105106
if ($SymbolsOnMSDL -eq $null -and $SymbolsOnSymWeb -eq $null) {
106-
Write-Host "No symbols found on MSDL or SymWeb!"
107+
Write-Host 'No symbols found on MSDL or SymWeb!'
107108
}
108109
else {
109110
if ($SymbolsOnMSDL -eq $null) {
110-
Write-Host "No symbols found on MSDL!"
111+
Write-Host 'No symbols found on MSDL!'
111112
}
112113
else {
113-
Write-Host "No symbols found on SymWeb!"
114+
Write-Host 'No symbols found on SymWeb!'
114115
}
115116
}
116117
}
@@ -129,26 +130,26 @@ function CheckSymbolsAvailable {
129130
Get-ChildItem "$InputPath\*.nupkg" |
130131
ForEach-Object {
131132
$FileName = $_.Name
132-
133+
133134
# These packages from Arcade-Services include some native libraries that
134135
# our current symbol uploader can't handle. Below is a workaround until
135136
# we get issue: https://github.com/dotnet/arcade/issues/2457 sorted.
136-
if ($FileName -Match "Microsoft\.DotNet\.Darc\.") {
137+
if ($FileName -Match 'Microsoft\.DotNet\.Darc\.') {
137138
Write-Host "Ignoring Arcade-services file: $FileName"
138139
Write-Host
139140
return
140141
}
141-
elseif ($FileName -Match "Microsoft\.DotNet\.Maestro\.Tasks\.") {
142+
elseif ($FileName -Match 'Microsoft\.DotNet\.Maestro\.Tasks\.') {
142143
Write-Host "Ignoring Arcade-services file: $FileName"
143144
Write-Host
144145
return
145146
}
146-
147+
147148
Write-Host "Validating $FileName "
148149
$Status = CountMissingSymbols "$InputPath\$FileName"
149150

150151
if ($Status -ne 0) {
151-
Write-Error "Missing symbols for $Status modules in the package $FileName"
152+
Write-PipelineTelemetryError -Category 'CheckSymbols' -Message "Missing symbols for $Status modules in the package $FileName"
152153
}
153154

154155
Write-Host
Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
@{
2+
IncludeRules=@('PSAvoidUsingCmdletAliases',
3+
'PSAvoidUsingWMICmdlet',
4+
'PSAvoidUsingPositionalParameters',
5+
'PSAvoidUsingInvokeExpression',
6+
'PSUseDeclaredVarsMoreThanAssignments',
7+
'PSUseCmdletCorrectly',
8+
'PSStandardDSCFunctionsInResource',
9+
'PSUseIdenticalMandatoryParametersForDSC',
10+
'PSUseIdenticalParametersForDSC')
11+
}

eng/common/PublishToPackageFeed.proj

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -53,6 +53,8 @@
5353
<TargetStaticFeed Condition="'$(ArtifactsCategory.ToUpper())' == 'TOOLSET'">https://dotnetfeed.blob.core.windows.net/dotnet-toolset/index.json</TargetStaticFeed>
5454
<TargetStaticFeed Condition="'$(ArtifactsCategory.ToUpper())' == 'WINDOWSDESKTOP'">https://dotnetfeed.blob.core.windows.net/dotnet-windowsdesktop/index.json</TargetStaticFeed>
5555
<TargetStaticFeed Condition="'$(ArtifactsCategory.ToUpper())' == 'NUGETCLIENT'">https://dotnetfeed.blob.core.windows.net/nuget-nugetclient/index.json</TargetStaticFeed>
56+
<TargetStaticFeed Condition="'$(ArtifactsCategory.ToUpper())' == 'ASPNETENTITYFRAMEWORK6'">https://dotnetfeed.blob.core.windows.net/aspnet-entityframework6/index.json</TargetStaticFeed>
57+
<TargetStaticFeed Condition="'$(ArtifactsCategory.ToUpper())' == 'ASPNETBLAZOR'">https://dotnetfeed.blob.core.windows.net/aspnet-blazor/index.json</TargetStaticFeed>
5658
</PropertyGroup>
5759

5860
<Error

eng/common/PublishToSymbolServers.proj

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -37,6 +37,8 @@
3737
<PropertyGroup>
3838
<DotNetSymbolExpirationInDays Condition="'$(DotNetSymbolExpirationInDays)' == ''">3650</DotNetSymbolExpirationInDays>
3939
<PublishToSymbolServer>true</PublishToSymbolServer>
40+
<PublishToSymWeb Condition="'$(PublishToSymWeb)' == ''">true</PublishToSymWeb>
41+
<PublishToMSDL Condition="'$(PublishToMSDL)' == ''">true</PublishToMSDL>
4042
<PublishToSymbolServer Condition="'@(FilesToPublishToSymbolServer)' == '' and '@(PackagesToPublishToSymbolServer)' == ''">false</PublishToSymbolServer>
4143
</PropertyGroup>
4244

@@ -56,7 +58,7 @@
5658
DryRun="false"
5759
ConvertPortablePdbsToWindowsPdbs="false"
5860
PdbConversionTreatAsWarning=""
59-
Condition="$(PublishToSymbolServer)"/>
61+
Condition="$(PublishToSymbolServer) and $(PublishToMSDL)"/>
6062

6163
<!--
6264
Symbol Uploader: SymWeb
@@ -73,7 +75,7 @@
7375
DryRun="false"
7476
ConvertPortablePdbsToWindowsPdbs="false"
7577
PdbConversionTreatAsWarning=""
76-
Condition="$(PublishToSymbolServer)"/>
78+
Condition="$(PublishToSymbolServer) and $(PublishToSymWeb)"/>
7779
</Target>
7880

7981
<ItemGroup>

eng/common/SetupNugetSources.ps1

Lines changed: 141 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,141 @@
1+
# This file is a temporary workaround for internal builds to be able to restore from private AzDO feeds.
2+
# This file should be removed as part of this issue: https://github.com/dotnet/arcade/issues/4080
3+
#
4+
# What the script does is iterate over all package sources in the pointed NuGet.config and add a credential entry
5+
# under <packageSourceCredentials> for each Maestro managed private feed. Two additional credential
6+
# entries are also added for the two private static internal feeds: dotnet3-internal and dotnet3-internal-transport.
7+
#
8+
# This script needs to be called in every job that will restore packages and which the base repo has
9+
# private AzDO feeds in the NuGet.config.
10+
#
11+
# See example YAML call for this script below. Note the use of the variable `$(dn-bot-dnceng-artifact-feeds-rw)`
12+
# from the AzureDevOps-Artifact-Feeds-Pats variable group.
13+
#
14+
# - task: PowerShell@2
15+
# displayName: Setup Private Feeds Credentials
16+
# condition: eq(variables['Agent.OS'], 'Windows_NT')
17+
# inputs:
18+
# filePath: $(Build.SourcesDirectory)/eng/common/SetupNugetSources.ps1
19+
# arguments: -ConfigFile $(Build.SourcesDirectory)/NuGet.config -Password $Env:Token
20+
# env:
21+
# Token: $(dn-bot-dnceng-artifact-feeds-rw)
22+
23+
[CmdletBinding()]
24+
param (
25+
[Parameter(Mandatory = $true)][string]$ConfigFile,
26+
[Parameter(Mandatory = $true)][string]$Password
27+
)
28+
29+
$ErrorActionPreference = "Stop"
30+
Set-StrictMode -Version 2.0
31+
[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12
32+
33+
. $PSScriptRoot\tools.ps1
34+
35+
# Add source entry to PackageSources
36+
function AddPackageSource($sources, $SourceName, $SourceEndPoint, $creds, $Username, $Password) {
37+
$packageSource = $sources.SelectSingleNode("add[@key='$SourceName']")
38+
39+
if ($packageSource -eq $null)
40+
{
41+
$packageSource = $doc.CreateElement("add")
42+
$packageSource.SetAttribute("key", $SourceName)
43+
$packageSource.SetAttribute("value", $SourceEndPoint)
44+
$sources.AppendChild($packageSource) | Out-Null
45+
}
46+
else {
47+
Write-Host "Package source $SourceName already present."
48+
}
49+
50+
AddCredential -Creds $creds -Source $SourceName -Username $Username -Password $Password
51+
}
52+
53+
# Add a credential node for the specified source
54+
function AddCredential($creds, $source, $username, $password) {
55+
# Looks for credential configuration for the given SourceName. Create it if none is found.
56+
$sourceElement = $creds.SelectSingleNode($Source)
57+
if ($sourceElement -eq $null)
58+
{
59+
$sourceElement = $doc.CreateElement($Source)
60+
$creds.AppendChild($sourceElement) | Out-Null
61+
}
62+
63+
# Add the <Username> node to the credential if none is found.
64+
$usernameElement = $sourceElement.SelectSingleNode("add[@key='Username']")
65+
if ($usernameElement -eq $null)
66+
{
67+
$usernameElement = $doc.CreateElement("add")
68+
$usernameElement.SetAttribute("key", "Username")
69+
$sourceElement.AppendChild($usernameElement) | Out-Null
70+
}
71+
$usernameElement.SetAttribute("value", $Username)
72+
73+
# Add the <ClearTextPassword> to the credential if none is found.
74+
# Add it as a clear text because there is no support for encrypted ones in non-windows .Net SDKs.
75+
# -> https://github.com/NuGet/Home/issues/5526
76+
$passwordElement = $sourceElement.SelectSingleNode("add[@key='ClearTextPassword']")
77+
if ($passwordElement -eq $null)
78+
{
79+
$passwordElement = $doc.CreateElement("add")
80+
$passwordElement.SetAttribute("key", "ClearTextPassword")
81+
$sourceElement.AppendChild($passwordElement) | Out-Null
82+
}
83+
$passwordElement.SetAttribute("value", $Password)
84+
}
85+
86+
function InsertMaestroPrivateFeedCredentials($Sources, $Creds, $Password) {
87+
$maestroPrivateSources = $Sources.SelectNodes("add[contains(@key,'darc-int')]")
88+
89+
Write-Host "Inserting credentials for $($maestroPrivateSources.Count) Maestro's private feeds."
90+
91+
ForEach ($PackageSource in $maestroPrivateSources) {
92+
Write-Host "`tInserting credential for Maestro's feed:" $PackageSource.Key
93+
AddCredential -Creds $creds -Source $PackageSource.Key -Username $Username -Password $Password
94+
}
95+
}
96+
97+
if (!(Test-Path $ConfigFile -PathType Leaf)) {
98+
Write-PipelineTelemetryError -Category 'Build' -Message "Eng/common/SetupNugetSources.ps1 returned a non-zero exit code. Couldn't find the NuGet config file: $ConfigFile"
99+
ExitWithExitCode 1
100+
}
101+
102+
if (!$Password) {
103+
Write-PipelineTelemetryError -Category 'Build' -Message 'Eng/common/SetupNugetSources.ps1 returned a non-zero exit code. Please supply a valid PAT'
104+
ExitWithExitCode 1
105+
}
106+
107+
# Load NuGet.config
108+
$doc = New-Object System.Xml.XmlDocument
109+
$filename = (Get-Item $ConfigFile).FullName
110+
$doc.Load($filename)
111+
112+
# Get reference to <PackageSources> or create one if none exist already
113+
$sources = $doc.DocumentElement.SelectSingleNode("packageSources")
114+
if ($sources -eq $null) {
115+
$sources = $doc.CreateElement("packageSources")
116+
$doc.DocumentElement.AppendChild($sources) | Out-Null
117+
}
118+
119+
# Looks for a <PackageSourceCredentials> node. Create it if none is found.
120+
$creds = $doc.DocumentElement.SelectSingleNode("packageSourceCredentials")
121+
if ($creds -eq $null) {
122+
$creds = $doc.CreateElement("packageSourceCredentials")
123+
$doc.DocumentElement.AppendChild($creds) | Out-Null
124+
}
125+
126+
# Insert credential nodes for Maestro's private feeds
127+
InsertMaestroPrivateFeedCredentials -Sources $sources -Creds $creds -Password $Password
128+
129+
$dotnet3Source = $sources.SelectSingleNode("add[@key='dotnet3']")
130+
if ($dotnet3Source -ne $null) {
131+
AddPackageSource -Sources $sources -SourceName "dotnet3-internal" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/_packaging/dotnet3-internal/nuget/v2" -Creds $creds -Username "dn-bot" -Password $Password
132+
AddPackageSource -Sources $sources -SourceName "dotnet3-internal-transport" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/_packaging/dotnet3-internal-transport/nuget/v2" -Creds $creds -Username "dn-bot" -Password $Password
133+
}
134+
135+
$dotnet31Source = $sources.SelectSingleNode("add[@key='dotnet3.1']")
136+
if ($dotnet31Source -ne $null) {
137+
AddPackageSource -Sources $sources -SourceName "dotnet3.1-internal" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/_packaging/dotnet3.1-internal/nuget/v2" -Creds $creds -Username "dn-bot" -Password $Password
138+
AddPackageSource -Sources $sources -SourceName "dotnet3.1-internal-transport" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/_packaging/dotnet3.1-internal-transport/nuget/v2" -Creds $creds -Username "dn-bot" -Password $Password
139+
}
140+
141+
$doc.Save($filename)

0 commit comments

Comments
 (0)