[main] Update dependencies from dotnet/arcade (#1234)

* Update dependencies from https://github.com/dotnet/arcade build 20210317.3

Microsoft.DotNet.Arcade.Sdk
 From Version 6.0.0-beta.21105.12 -> To Version 6.0.0-beta.21167.3

* remain on stable sdk

Co-authored-by: dotnet-maestro[bot] <dotnet-maestro[bot]@users.noreply.github.com>
Co-authored-by: Brett V. Forsgren <[email protected]>

Author: dotnet-maestro[bot]

eng/Version.Details.xml

@@ -3,10 +3,10 @@
   <ProductDependencies>
   </ProductDependencies>
   <ToolsetDependencies>
-    <Dependency Name="Microsoft.DotNet.Arcade.Sdk" Version="6.0.0-beta.21105.12">
+    <Dependency Name="Microsoft.DotNet.Arcade.Sdk" Version="6.0.0-beta.21167.3">
       <Uri>https://github.com/dotnet/arcade</Uri>
-      <Sha>938b3e8b4edcd96ca0f0cbbae63c87b3f51f7afe</Sha>
-      <SourceBuild RepoName="arcade" ManagedOnly="true"/>
+      <Sha>0ca849f0b71866b007fedaaa938cee63f8d056a6</Sha>
+      <SourceBuild RepoName="arcade" ManagedOnly="true" />
     </Dependency>
   </ToolsetDependencies>
 </Dependencies>

eng/common/cross/armel/tizen-fetch.sh

@@ -157,7 +157,7 @@ fetch_tizen_pkgs()
 Inform "Initialize arm base"
 fetch_tizen_pkgs_init standard base
 Inform "fetch common packages"
-fetch_tizen_pkgs armv7l gcc glibc glibc-devel libicu libicu-devel libatomic linux-glibc-devel
+fetch_tizen_pkgs armv7l gcc gcc-devel-static glibc glibc-devel libicu libicu-devel libatomic linux-glibc-devel keyutils keyutils-devel libkeyutils
 Inform "fetch coreclr packages"
 fetch_tizen_pkgs armv7l lldb lldb-devel libgcc libstdc++ libstdc++-devel libunwind libunwind-devel lttng-ust-devel lttng-ust userspace-rcu-devel userspace-rcu
 Inform "fetch corefx packages"

eng/common/cross/build-rootfs.sh

@@ -6,7 +6,7 @@ usage()
 {
     echo "Usage: $0 [BuildArch] [CodeName] [lldbx.y] [--skipunmount] --rootfsdir <directory>]"
     echo "BuildArch can be: arm(default), armel, arm64, x86"
-    echo "CodeName - optional, Code name for Linux, can be: trusty, xenial(default), zesty, bionic, alpine. If BuildArch is armel, LinuxCodeName is jessie(default) or tizen."
+    echo "CodeName - optional, Code name for Linux, can be: trusty, xenial(default), zesty, bionic, alpine, alpine3.9 or alpine3.13. If BuildArch is armel, LinuxCodeName is jessie(default) or tizen."
     echo "                              for FreeBSD can be: freebsd11 or freebsd12."
     echo "                              for illumos can be: illumos."
     echo "lldbx.y - optional, LLDB version, can be: lldb3.9(default), lldb4.0, lldb5.0, lldb6.0 no-lldb. Ignored for alpine and FReeBSD"
@@ -183,9 +183,20 @@ while :; do
             __UbuntuRepo=
             __Tizen=tizen
             ;;
-        alpine)
+        alpine|alpine3.9)
             __CodeName=alpine
             __UbuntuRepo=
+            __AlpineVersion=3.9
+            ;;
+        alpine3.13)
+            __CodeName=alpine
+            __UbuntuRepo=
+            __AlpineVersion=3.13
+            # Alpine 3.13 has all the packages we need in the 3.13 repository
+            __AlpinePackages+=$__AlpinePackagesEdgeCommunity
+            __AlpinePackagesEdgeCommunity=
+            __AlpinePackages+=$__AlpinePackagesEdgeMain
+            __AlpinePackagesEdgeMain=
             ;;
         freebsd11)
             __FreeBSDBase="11.3-RELEASE"
@@ -243,7 +254,6 @@ __RootfsDir="$( cd "$__RootfsDir" && pwd )"
 
 if [[ "$__CodeName" == "alpine" ]]; then
     __ApkToolsVersion=2.9.1
-    __AlpineVersion=3.9
     __ApkToolsDir=$(mktemp -d)
     wget https://github.com/alpinelinux/apk-tools/releases/download/v$__ApkToolsVersion/apk-tools-$__ApkToolsVersion-x86_64-linux.tar.gz -P $__ApkToolsDir
     tar -xf $__ApkToolsDir/apk-tools-$__ApkToolsVersion-x86_64-linux.tar.gz -C $__ApkToolsDir
@@ -256,15 +266,19 @@ if [[ "$__CodeName" == "alpine" ]]; then
       -U --allow-untrusted --root $__RootfsDir --arch $__AlpineArch --initdb \
       add $__AlpinePackages
 
-    $__ApkToolsDir/apk-tools-$__ApkToolsVersion/apk \
-      -X http://dl-cdn.alpinelinux.org/alpine/edge/main \
-      -U --allow-untrusted --root $__RootfsDir --arch $__AlpineArch --initdb \
-      add $__AlpinePackagesEdgeMain
+    if [[ -n "$__AlpinePackagesEdgeMain" ]]; then
+      $__ApkToolsDir/apk-tools-$__ApkToolsVersion/apk \
+        -X http://dl-cdn.alpinelinux.org/alpine/edge/main \
+        -U --allow-untrusted --root $__RootfsDir --arch $__AlpineArch --initdb \
+        add $__AlpinePackagesEdgeMain
+    fi
 
-    $__ApkToolsDir/apk-tools-$__ApkToolsVersion/apk \
-      -X http://dl-cdn.alpinelinux.org/alpine/edge/community \
-      -U --allow-untrusted --root $__RootfsDir --arch $__AlpineArch --initdb \
-      add $__AlpinePackagesEdgeCommunity
+    if [[ -n "$__AlpinePackagesEdgeCommunity" ]]; then
+      $__ApkToolsDir/apk-tools-$__ApkToolsVersion/apk \
+        -X http://dl-cdn.alpinelinux.org/alpine/edge/community \
+        -U --allow-untrusted --root $__RootfsDir --arch $__AlpineArch --initdb \
+        add $__AlpinePackagesEdgeCommunity
+    fi
 
     rm -r $__ApkToolsDir
 elif [[ "$__CodeName" == "freebsd" ]]; then

eng/common/generate-locproject.ps1

@@ -0,0 +1,101 @@
+Param(
+    [Parameter(Mandatory=$true)][string] $SourcesDirectory,     # Directory where source files live; if using a Localize directory it should live in here
+    [string] $LanguageSet = 'VS_Main_Languages',                # Language set to be used in the LocProject.json
+    [switch] $UseCheckedInLocProjectJson,                       # When set, generates a LocProject.json and compares it to one that already exists in the repo; otherwise just generates one
+    [switch] $CreateNeutralXlfs                                 # Creates neutral xlf files. Only set to false when running locally
+)
+
+# Generates LocProject.json files for the OneLocBuild task. OneLocBuildTask is described here:
+# https://ceapex.visualstudio.com/CEINTL/_wiki/wikis/CEINTL.wiki/107/Localization-with-OneLocBuild-Task
+
+Set-StrictMode -Version 2.0
+$ErrorActionPreference = "Stop"
+. $PSScriptRoot\tools.ps1
+
+Import-Module -Name (Join-Path $PSScriptRoot 'native\CommonLibrary.psm1')
+
+$exclusionsFilePath = "$SourcesDirectory\Localize\LocExclusions.json"
+$exclusions = @{ Exclusions = @() }
+if (Test-Path -Path $exclusionsFilePath)
+{
+    $exclusions = Get-Content "$exclusionsFilePath" | ConvertFrom-Json
+}
+
+Push-Location "$SourcesDirectory" # push location for Resolve-Path -Relative to work
+
+# Template files
+$jsonFiles = @()
+$jsonFiles += Get-ChildItem -Recurse -Path "$SourcesDirectory" | Where-Object { $_.FullName -Match "\.template\.config\\localize\\en\..+\.json" } # .NET templating pattern
+$jsonFiles += Get-ChildItem -Recurse -Path "$SourcesDirectory" | Where-Object { $_.FullName -Match "en\\strings\.json" } # current winforms pattern
+
+$xlfFiles = @()
+
+$allXlfFiles = Get-ChildItem -Recurse -Path "$SourcesDirectory\*\*.xlf"
+$langXlfFiles = @()
+if ($allXlfFiles) {
+    $null = $allXlfFiles[0].FullName -Match "\.([\w-]+)\.xlf" # matches '[langcode].xlf'
+    $firstLangCode = $Matches.1
+    $langXlfFiles = Get-ChildItem -Recurse -Path "$SourcesDirectory\*\*.$firstLangCode.xlf"
+}
+$langXlfFiles | ForEach-Object {
+    $null = $_.Name -Match "([^.]+)\.[\w-]+\.xlf" # matches '[filename].[langcode].xlf'
+    
+    $destinationFile = "$($_.Directory.FullName)\$($Matches.1).xlf"
+    $xlfFiles += Copy-Item "$($_.FullName)" -Destination $destinationFile -PassThru
+}
+
+$locFiles = $jsonFiles + $xlfFiles
+
+$locJson = @{
+    Projects = @(
+        @{
+            LanguageSet = $LanguageSet
+            LocItems = @(
+                $locFiles | ForEach-Object {
+                    $outputPath = "Localize\$(($_.DirectoryName | Resolve-Path -Relative) + "\")" 
+                    $continue = $true
+                    foreach ($exclusion in $exclusions.Exclusions) {
+                        if ($outputPath.Contains($exclusion))
+                        {
+                            $continue = $false
+                        }
+                    }
+                    $sourceFile = ($_.FullName | Resolve-Path -Relative)
+                    if (!$CreateNeutralXlfs -and $_.Extension -eq '.xlf') {
+                        Remove-Item -Path $sourceFile
+                    }
+                    if ($continue)
+                    {
+                        return @{
+                            SourceFile = $sourceFile
+                            CopyOption = "LangIDOnName"
+                            OutputPath = $outputPath
+                        }
+                    }
+                }
+            )
+        }
+    )
+}
+
+$json = ConvertTo-Json $locJson -Depth 5
+Write-Host "(NETCORE_ENGINEERING_TELEMETRY=Build) LocProject.json generated:`n`n$json`n`n"
+Pop-Location
+
+if (!$UseCheckedInLocProjectJson) {
+    New-Item "$SourcesDirectory\Localize\LocProject.json" -Force # Need this to make sure the Localize directory is created
+    Set-Content "$SourcesDirectory\Localize\LocProject.json" $json
+}
+else {
+    New-Item "$SourcesDirectory\Localize\LocProject-generated.json" -Force # Need this to make sure the Localize directory is created
+    Set-Content "$SourcesDirectory\Localize\LocProject-generated.json" $json
+
+    if ((Get-FileHash "$SourcesDirectory\Localize\LocProject-generated.json").Hash -ne (Get-FileHash "$SourcesDirectory\Localize\LocProject.json").Hash) {
+        Write-PipelineTaskError -Type "warning" -Message "Existing LocProject.json differs from generated LocProject.json. Download LocProject-generated.json and compare them."
+        
+        exit 1
+    }
+    else {
+        Write-Host "Generated LocProject.json and current LocProject.json are identical."
+    }
+}

eng/common/native/install-tool.ps1

@@ -105,7 +105,7 @@ try {
     Write-Error "There are multiple copies of $ToolName in $($ToolInstallDirectory): `n$(@($ToolFilePath | out-string))"
     exit 1
   } elseif (@($ToolFilePath).Length -Lt 1) {
-    Write-Host "$ToolName was not found in $ToolFilePath."
+    Write-Host "$ToolName was not found in $ToolInstallDirectory."
     exit 1
   }
 

eng/common/performance/performance-setup.ps1

@@ -18,7 +18,8 @@ Param(
     [switch] $Internal,
     [switch] $Compare,
     [string] $MonoDotnet="",
-    [string] $Configurations="CompilationMode=$CompilationMode RunKind=$Kind"
+    [string] $Configurations="CompilationMode=$CompilationMode RunKind=$Kind",
+    [string] $LogicalMachine=""
 )
 
 $RunFromPerformanceRepo = ($Repository -eq "dotnet/performance") -or ($Repository -eq "dotnet-performance")
@@ -33,26 +34,23 @@ $Creator = $env:BUILD_DEFINITIONNAME
 $PerfLabArguments = ""
 $HelixSourcePrefix = "pr"
 
-$Queue = "Windows.10.Amd64.ClientRS4.DevEx.15.8.Open"
-
-# TODO: Implement a better logic to determine if Framework is .NET Core or >= .NET 5.
-if ($Framework.StartsWith("netcoreapp") -or ($Framework -eq "net5.0")) {
-    $Queue = "Windows.10.Amd64.ClientRS5.Open"
-}
-
-if ($Compare) {
-    $Queue = "Windows.10.Amd64.19H1.Tiger.Perf.Open"
-    $PerfLabArguments = ""
-    $ExtraBenchmarkDotNetArguments = ""
-}
+$Queue = ""
 
 if ($Internal) {
-    $Queue = "Windows.10.Amd64.19H1.Tiger.Perf"
+    switch ($LogicalMachine) {
+        "perftiger" { $Queue = "Windows.10.Amd64.19H1.Tiger.Perf"  }
+        "perfowl" { $Queue = "Windows.10.Amd64.20H2.Owl.Perf"  }
+        "perfsurf" { $Queue = "Windows.10.Arm64.Perf.Surf"  }
+        Default { $Queue = "Windows.10.Amd64.19H1.Tiger.Perf" }
+    }
     $PerfLabArguments = "--upload-to-perflab-container"
     $ExtraBenchmarkDotNetArguments = ""
     $Creator = ""
     $HelixSourcePrefix = "official"
 }
+else {
+    $Queue = "Windows.10.Amd64.ClientRS4.DevEx.15.8.Open"
+}
 
 if($MonoInterpreter)
 {
@@ -79,14 +77,6 @@ $CommonSetupArguments="--channel master --queue $Queue --build-number $BuildNumb
 $SetupArguments = "--repository https://github.com/$Repository --branch $Branch --get-perf-hash --commit-sha $CommitSha $CommonSetupArguments"
 
 
-#This grabs the LKG version number of dotnet and passes it to our scripts
-$VersionJSON = Get-Content global.json | ConvertFrom-Json
-$DotNetVersion = $VersionJSON.tools.dotnet
-# TODO: Change this back to parsing when we have a good story for dealing with TFM changes or when the LKG in runtime gets updated to include net6.0
-# $SetupArguments = "--dotnet-versions $DotNetVersion $SetupArguments"
-$SetupArguments = "--dotnet-versions 6.0.100-alpha.1.20553.6 $SetupArguments"
-
-
 if ($RunFromPerformanceRepo) {
     $SetupArguments = "--perf-hash $CommitSha $CommonSetupArguments"
 

eng/common/performance/performance-setup.sh

@@ -27,6 +27,7 @@ using_mono=false
 wasm_runtime_loc=
 using_wasm=false
 use_latest_dotnet=false
+logical_machine=
 
 while (($# > 0)); do
   lowerI="$(echo $1 | tr "[:upper:]" "[:lower:]")"
@@ -55,6 +56,10 @@ while (($# > 0)); do
       compilation_mode=$2
       shift 2
       ;;
+    --logicalmachine)
+      logical_machine=$2
+      shift 2
+      ;;
     --repository)
       repository=$2
       shift 2
@@ -88,6 +93,10 @@ while (($# > 0)); do
       internal=true
       shift 1
       ;;
+    --alpine)
+      alpine=true
+      shift 1
+      ;;
     --llvm)
       llvm=true
       shift 1
@@ -143,6 +152,7 @@ while (($# > 0)); do
       echo "  --monodotnet                   Pass the path to the mono dotnet for mono performance testing."
       echo "  --wasm                         Path to the unpacked wasm runtime pack."
       echo "  --latestdotnet                 --dotnet-versions will not be specified. --dotnet-versions defaults to LKG version in global.json "
+      echo "  --alpine                       Set for runs on Alpine"
       echo ""
       exit 0
       ;;
@@ -174,19 +184,6 @@ queue=Ubuntu.1804.Amd64.Open
 creator=$BUILD_DEFINITIONNAME
 helix_source_prefix="pr"
 
-if [[ "$compare" == true ]]; then
-  extra_benchmark_dotnet_arguments=
-  perflab_arguments=
-
-  # No open queues for arm64
-  if [[ "$architecture" = "arm64" ]]; then
-    echo "Compare not available for arm64"
-    exit 1
-  fi
-
-  queue=Ubuntu.1804.Amd64.Tiger.Perf.Open
-fi
-
 if [[ "$internal" == true ]]; then
     perflab_arguments="--upload-to-perflab-container"
     helix_source_prefix="official"
@@ -196,14 +193,26 @@ if [[ "$internal" == true ]]; then
     if [[ "$architecture" = "arm64" ]]; then
         queue=Ubuntu.1804.Arm64.Perf
     else
-        queue=Ubuntu.1804.Amd64.Tiger.Perf
+        if [[ "$logical_machine" = "perfowl" ]]; then
+            queue=Ubuntu.1804.Amd64.Owl.Perf
+        else
+            queue=Ubuntu.1804.Amd64.Tiger.Perf
+        fi
+    fi
+
+    if [[ "$alpine" = "true" ]]; then
+        queue=alpine.amd64.tiger.perf
     fi
 else
     if [[ "$architecture" = "arm64" ]]; then
         queue=ubuntu.1804.armarch.open
     else
         queue=Ubuntu.1804.Amd64.Open
     fi
+
+    if [[ "$alpine" = "true" ]]; then
+        queue=alpine.amd64.tiger.perf
+    fi
 fi
 
 if [[ "$mono_dotnet" != "" ]] && [[ "$monointerpreter" == "false" ]]; then
@@ -224,16 +233,6 @@ fi
 common_setup_arguments="--channel master --queue $queue --build-number $build_number --build-configs $configurations --architecture $architecture"
 setup_arguments="--repository https://github.com/$repository --branch $branch --get-perf-hash --commit-sha $commit_sha $common_setup_arguments"
 
-
-if [[ "$use_latest_dotnet" = false ]]; then
-    # Get the tools section from the global.json.
-    # This grabs the LKG version number of dotnet and passes it to our scripts
-    dotnet_version=`cat global.json | python3 -c 'import json,sys;obj=json.load(sys.stdin);print(obj["tools"]["dotnet"])'`
-    # TODO: Change this back to parsing when we have a good story for dealing with TFM changes or when the LKG in runtime gets updated to include net6.0
-    # setup_arguments="--dotnet-versions $dotnet_version $setup_arguments"
-    setup_arguments="--dotnet-versions 6.0.100-alpha.1.20553.6 $setup_arguments"
-fi
-
 if [[ "$run_from_perf_repo" = true ]]; then
     payload_directory=
     workitem_directory=$source_directory

eng/common/post-build/publish-using-darc.ps1

@@ -60,7 +60,7 @@ try {
   --id $buildId `
   --publishing-infra-version $PublishingInfraVersion `
   --default-channels `
-  --source-branch master `
+  --source-branch main `
   --azdev-pat $AzdoToken `
   --bar-uri $MaestroApiEndPoint `
   --password $MaestroToken `

eng/common/sdl/execute-all-sdl-tools.ps1

@@ -87,10 +87,6 @@ try {
     & $(Join-Path $PSScriptRoot 'run-sdl.ps1') -GuardianCliLocation $guardianCliLocation -WorkingDirectory $workingDirectory -TargetDirectory $SourceDirectory -GdnFolder $gdnFolder -ToolsList $SourceToolsList -AzureDevOpsAccessToken $AzureDevOpsAccessToken -UpdateBaseline $UpdateBaseline -GuardianLoggerLevel $GuardianLoggerLevel -CrScanAdditionalRunConfigParams $CrScanAdditionalRunConfigParams -PoliCheckAdditionalRunConfigParams $PoliCheckAdditionalRunConfigParams
   }
 
-  if ($UpdateBaseline) {
-    & (Join-Path $PSScriptRoot 'push-gdn.ps1') -Repository $RepoName -BranchName $BranchName -GdnFolder $GdnFolder -AzureDevOpsAccessToken $AzureDevOpsAccessToken -PushReason 'Update baseline'
-  }
-
   if ($TsaPublish) {
     if ($TsaBranchName -and $BuildNumber) {
       if (-not $TsaRepositoryName) {

eng/common/sdl/init-sdl.ps1

@@ -46,7 +46,6 @@ try {
     Write-PipelineTelemetryError -Force -Category 'Build' -Message "Guardian baseline failed with exit code $LASTEXITCODE."
     ExitWithExitCode $LASTEXITCODE
   }
-  & $(Join-Path $PSScriptRoot 'push-gdn.ps1') -Repository $Repository -BranchName $BranchName -GdnFolder $gdnFolder -AzureDevOpsAccessToken $AzureDevOpsAccessToken -PushReason 'Initialize gdn folder'
   ExitWithExitCode 0
 }
 catch {