Merge branch 'main' into unify-test-utilities

Author: DustinCampbell

azure-pipelines.yml

@@ -83,10 +83,10 @@ stages:
           pool:
             ${{ if eq(variables['System.TeamProject'], 'public') }}:
               name: $(DncEngPublicBuildPool)
-              demands: ImageOverride -equals windows.vs2022preview.scout.amd64.open
+              demands: ImageOverride -equals windows.vs2022preview.amd64.open
             ${{ if ne(variables['System.TeamProject'], 'public') }}:
               name: $(DncEngInternalBuildPool)
-              demands: ImageOverride -equals windows.vs2022preview.scout.amd64
+              demands: ImageOverride -equals windows.vs2022preview.amd64
           steps:
           - task: NuGetCommand@2
             displayName: 'Clear NuGet caches'
@@ -127,10 +127,10 @@ stages:
         pool:
           ${{ if eq(variables['System.TeamProject'], 'public') }}:
             name: $(DncEngPublicBuildPool)
-            demands: ImageOverride -equals windows.vs2022preview.scout.amd64.open
+            demands: ImageOverride -equals windows.vs2022preview.amd64.open
           ${{ if ne(variables['System.TeamProject'], 'public') }}:
             name: $(DncEngInternalBuildPool)
-            demands: ImageOverride -equals windows.vs2022preview.scout.amd64
+            demands: ImageOverride -equals windows.vs2022preview.amd64
         strategy:
           matrix:
             ${{ if eq(variables['System.TeamProject'], 'public') }}:

eng/Version.Details.xml

@@ -94,14 +94,14 @@
       <Uri>https://dev.azure.com/dnceng/internal/_git/dotnet-runtime</Uri>
       <Sha>3a25a7f1cc446b60678ed25c9d829420d6321eba</Sha>
     </Dependency>
-    <Dependency Name="Microsoft.DotNet.Arcade.Sdk" Version="9.0.0-beta.23572.2">
+    <Dependency Name="Microsoft.DotNet.Arcade.Sdk" Version="9.0.0-beta.23601.1">
       <Uri>https://github.com/dotnet/arcade</Uri>
-      <Sha>c5e5051496efbcc9046b25628666377d6b97a670</Sha>
+      <Sha>38aa367e014493c6f6ebfe94d9029bea01d072c4</Sha>
       <SourceBuild RepoName="arcade" ManagedOnly="true" />
     </Dependency>
-    <Dependency Name="Microsoft.DotNet.XliffTasks" Version="9.0.0-beta.23572.2">
+    <Dependency Name="Microsoft.DotNet.XliffTasks" Version="9.0.0-beta.23601.1">
       <Uri>https://github.com/dotnet/arcade</Uri>
-      <Sha>c5e5051496efbcc9046b25628666377d6b97a670</Sha>
+      <Sha>38aa367e014493c6f6ebfe94d9029bea01d072c4</Sha>
     </Dependency>
     <!-- Necessary for source-build. This allows Microsoft.Extensions.ObjectPool and System.Collections.Immutable packages
          to be retrieved from live source-build and their content consumed by packages produced by razor.

eng/Versions.props

@@ -71,7 +71,7 @@
     <MicrosoftCodeAnalysisWorkspacesCommonPackageVersion>4.8.0-3.23475.1</MicrosoftCodeAnalysisWorkspacesCommonPackageVersion>
     <MicrosoftCodeAnalysisWorkspacesMSBuildPackageVersion>4.8.0-3.23475.1</MicrosoftCodeAnalysisWorkspacesMSBuildPackageVersion>
     <MicrosoftVisualStudioLanguageServicesPackageVersion>4.8.0-3.23475.1</MicrosoftVisualStudioLanguageServicesPackageVersion>
-    <MicrosoftDotNetXliffTasksPackageVersion>9.0.0-beta.23572.2</MicrosoftDotNetXliffTasksPackageVersion>
+    <MicrosoftDotNetXliffTasksPackageVersion>9.0.0-beta.23601.1</MicrosoftDotNetXliffTasksPackageVersion>
     <!--
       Exception - Microsoft.Extensions.ObjectPool and System.Collections.Immutable packages are not updated by automation,
       but are present in Version.Details.xml for source-build PVP flow. See the comment in Version.Details.xml for more information.

eng/common/darc-init.ps1

@@ -1,6 +1,6 @@
 param (
     $darcVersion = $null,
-    $versionEndpoint = 'https://maestro-prod.westus2.cloudapp.azure.com/api/assets/darc-version?api-version=2019-01-16',
+    $versionEndpoint = 'https://maestro.dot.net/api/assets/darc-version?api-version=2019-01-16',
     $verbosity = 'minimal',
     $toolpath = $null
 )

eng/common/darc-init.sh

@@ -2,7 +2,7 @@
 
 source="${BASH_SOURCE[0]}"
 darcVersion=''
-versionEndpoint='https://maestro-prod.westus2.cloudapp.azure.com/api/assets/darc-version?api-version=2019-01-16'
+versionEndpoint='https://maestro.dot.net/api/assets/darc-version?api-version=2019-01-16'
 verbosity='minimal'
 
 while [[ $# > 0 ]]; do

eng/common/post-build/add-build-to-channel.ps1

@@ -2,7 +2,7 @@ param(
   [Parameter(Mandatory=$true)][int] $BuildId,
   [Parameter(Mandatory=$true)][int] $ChannelId,
   [Parameter(Mandatory=$true)][string] $MaestroApiAccessToken,
-  [Parameter(Mandatory=$false)][string] $MaestroApiEndPoint = 'https://maestro-prod.westus2.cloudapp.azure.com',
+  [Parameter(Mandatory=$false)][string] $MaestroApiEndPoint = 'https://maestro.dot.net',
   [Parameter(Mandatory=$false)][string] $MaestroApiVersion = '2019-01-16'
 )
 

eng/common/post-build/publish-using-darc.ps1

@@ -3,7 +3,7 @@ param(
   [Parameter(Mandatory=$true)][int] $PublishingInfraVersion,
   [Parameter(Mandatory=$true)][string] $AzdoToken,
   [Parameter(Mandatory=$true)][string] $MaestroToken,
-  [Parameter(Mandatory=$false)][string] $MaestroApiEndPoint = 'https://maestro-prod.westus2.cloudapp.azure.com',
+  [Parameter(Mandatory=$false)][string] $MaestroApiEndPoint = 'https://maestro.dot.net',
   [Parameter(Mandatory=$true)][string] $WaitPublishingFinish,
   [Parameter(Mandatory=$false)][string] $ArtifactsPublishingAdditionalParameters,
   [Parameter(Mandatory=$false)][string] $SymbolPublishingAdditionalParameters

eng/common/post-build/redact-logs.ps1

@@ -0,0 +1,81 @@
+[CmdletBinding(PositionalBinding=$False)]
+param(
+  [Parameter(Mandatory=$true, Position=0)][string] $InputPath,
+  [Parameter(Mandatory=$true)][string] $BinlogToolVersion,
+  [Parameter(Mandatory=$false)][string] $DotnetPath,
+  [Parameter(Mandatory=$false)][string] $PackageFeed = 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-public/nuget/v3/index.json',
+  # File with strings to redact - separated by newlines.
+  #  For comments start the line with '# ' - such lines are ignored 
+  [Parameter(Mandatory=$false)][string] $TokensFilePath,
+  [Parameter(ValueFromRemainingArguments=$true)][String[]]$TokensToRedact
+)
+
+try {
+  . $PSScriptRoot\post-build-utils.ps1
+
+  $packageName = 'binlogtool'
+
+  $dotnet = $DotnetPath
+
+  if (!$dotnet) {
+    $dotnetRoot = InitializeDotNetCli -install:$true
+    $dotnet = "$dotnetRoot\dotnet.exe"
+  }
+  
+  $toolList = & "$dotnet" tool list -g
+
+  if ($toolList -like "*$packageName*") {
+    & "$dotnet" tool uninstall $packageName -g
+  }
+
+  $toolPath  = "$PSScriptRoot\..\..\..\.tools"
+  $verbosity = 'minimal'
+  
+  New-Item -ItemType Directory -Force -Path $toolPath
+  
+  Push-Location -Path $toolPath
+
+  try {
+    Write-Host "Installing Binlog redactor CLI..."
+    Write-Host "'$dotnet' new tool-manifest"
+    & "$dotnet" new tool-manifest
+    Write-Host "'$dotnet' tool install $packageName --local --add-source '$PackageFeed' -v $verbosity --version $BinlogToolVersion"
+    & "$dotnet" tool install $packageName --local --add-source "$PackageFeed" -v $verbosity --version $BinlogToolVersion
+
+    if (Test-Path $TokensFilePath) {
+        Write-Host "Adding additional sensitive data for redaction from file: " $TokensFilePath
+        $TokensToRedact += Get-Content -Path $TokensFilePath | Foreach {$_.Trim()} | Where { $_ -notmatch "^# " }
+    }
+
+    $optionalParams = [System.Collections.ArrayList]::new()
+  
+    Foreach ($p in $TokensToRedact)
+    {
+      if($p -match '^\$\(.*\)$')
+      {
+        Write-Host ("Ignoring token {0} as it is probably unexpanded AzDO variable"  -f $p)
+      }          
+      elseif($p)
+      {
+        $optionalParams.Add("-p:" + $p) | Out-Null
+      }
+    }
+
+    & $dotnet binlogtool redact --input:$InputPath --recurse --in-place `
+      @optionalParams
+
+    if ($LastExitCode -ne 0) {
+      Write-PipelineTelemetryError -Category 'Redactor' -Type 'warning' -Message "Problems using Redactor tool (exit code: $LastExitCode). But ignoring them now."
+    }
+  }
+  finally {
+    Pop-Location
+  }
+
+  Write-Host 'done.'
+} 
+catch {
+  Write-Host $_
+  Write-PipelineTelemetryError -Category 'Redactor' -Message "There was an error while trying to redact logs. Error: $_"
+  ExitWithExitCode 1
+}

eng/common/post-build/trigger-subscriptions.ps1

@@ -2,7 +2,7 @@ param(
   [Parameter(Mandatory=$true)][string] $SourceRepo,
   [Parameter(Mandatory=$true)][int] $ChannelId,
   [Parameter(Mandatory=$true)][string] $MaestroApiAccessToken,
-  [Parameter(Mandatory=$false)][string] $MaestroApiEndPoint = 'https://maestro-prod.westus2.cloudapp.azure.com',
+  [Parameter(Mandatory=$false)][string] $MaestroApiEndPoint = 'https://maestro.dot.net',
   [Parameter(Mandatory=$false)][string] $MaestroApiVersion = '2019-01-16'
 )
 

eng/common/templates/job/publish-build-assets.yml

@@ -48,8 +48,8 @@ jobs:
     - group: AzureDevOps-Artifact-Feeds-Pats
     - name: runCodesignValidationInjection
       value: false
-    - ${{ if eq(parameters.publishAssetsImmediately, 'true') }}:
-      - template: /eng/common/templates/post-build/common-variables.yml
+    # unconditional - needed for logs publishing (redactor tool version)
+    - template: /eng/common/templates/post-build/common-variables.yml
 
   pool:
     # We don't use the collection uri here because it might vary (.visualstudio.com vs. dev.azure.com)
@@ -81,7 +81,7 @@ jobs:
         arguments: -task PublishBuildAssets -restore -msbuildEngine dotnet
           /p:ManifestsPath='$(Build.StagingDirectory)/Download/AssetManifests'
           /p:BuildAssetRegistryToken=$(MaestroAccessToken)
-          /p:MaestroApiEndpoint=https://maestro-prod.westus2.cloudapp.azure.com
+          /p:MaestroApiEndpoint=https://maestro.dot.net
           /p:PublishUsingPipelines=${{ parameters.publishUsingPipelines }}
           /p:OfficialBuildId=$(Build.BuildNumber)
       condition: ${{ parameters.condition }}