[release/6.0] Switch to dSAS for internal runtimes (#56487)

* Update arcade

* Switch to dSAS for internal runtimes

* Remove cred templates from public jobs

* Enable internal sources for source build

Author: mmitche

.azure/pipelines/ci.yml

@@ -75,7 +75,6 @@ variables:
     value: /bl:artifacts/log/Release/Build.Installers.binlog
   - name: WindowsArm64InstallersLogArgs
     value: /bl:artifacts/log/Release/Build.Installers.Arm64.binlog
-- group: DotNetBuilds storage account read tokens
 - name: _InternalRuntimeDownloadArgs
   value: -RuntimeSourceFeed https://dotnetbuilds.blob.core.windows.net/internal
          -RuntimeSourceFeedKey $(dotnetbuilds-internal-container-read-token-base64)
@@ -750,6 +749,7 @@ extends:
       # Source build
       - template: /eng/common/templates-official/job/source-build.yml@self
         parameters:
+          enableInternalSources: true
           platform:
             name: 'Managed'
             container: 'mcr.microsoft.com/dotnet-buildtools/prereqs:centos-7'

.azure/pipelines/jobs/default-build.yml

@@ -199,7 +199,6 @@ jobs:
                 # Include the variables we always want.
                 COMPlus_DbgEnableMiniDump: 1
                 COMPlus_DbgMiniDumpName: "$(System.DefaultWorkingDirectory)/dotnet-%d.%t.core"
-                DotNetBuildsInternalReadSasToken: $(dotnetbuilds-internal-container-read-token)
                 # Expand provided `env:` properties, if any.
                 ${{ if step.env }}:
                   ${{ step.env }}
@@ -210,14 +209,12 @@ jobs:
             env:
               COMPlus_DbgEnableMiniDump: 1
               COMPlus_DbgMiniDumpName: "$(System.DefaultWorkingDirectory)/dotnet-%d.%t.core"
-              DotNetBuildsInternalReadSasToken: $(dotnetbuilds-internal-container-read-token)
         - ${{ if ne(parameters.agentOs, 'Windows') }}:
           - script: $(BuildDirectory)/build.sh --ci --nobl --configuration $(BuildConfiguration) $(BuildScriptArgs)
             displayName: Run build.sh
             env:
               COMPlus_DbgEnableMiniDump: 1
               COMPlus_DbgMiniDumpName: "$(System.DefaultWorkingDirectory)/dotnet-%d.%t.core"
-              DotNetBuildsInternalReadSasToken: $(dotnetbuilds-internal-container-read-token)
 
       - ${{ parameters.afterBuild }}
 
@@ -430,6 +427,20 @@ jobs:
           env:
             Token: $(dn-bot-dnceng-artifact-feeds-rw)
 
+      # Populates internal runtime SAS tokens.
+      - template: /eng/common/templates-official/steps/enable-internal-runtimes.yml
+
+      # Populate dotnetbuilds-internal base64 sas tokens.
+      - template: /eng/common/templates-official/steps/get-delegation-sas.yml
+        parameters:
+          federatedServiceConnection: 'dotnetbuilds-internal-read'
+          outputVariableName: 'dotnetbuilds-internal-container-read-token'
+          expiryInHours: 1
+          base64Encode: false
+          storageAccount: dotnetbuilds
+          container: internal
+          permissions: rl
+
       # Add COMPlus_* environment variables to build steps.
       - ${{ if ne(parameters.steps, '')}}:
         - ${{ each step in parameters.steps }}:

eng/Version.Details.xml

@@ -280,22 +280,22 @@
       <Uri>https://dev.azure.com/dnceng/internal/_git/dotnet-runtime</Uri>
       <Sha>1b9707a1fa56abb49ac8a286fd89a91527b6c7d3</Sha>
     </Dependency>
-    <Dependency Name="Microsoft.DotNet.Arcade.Sdk" Version="6.0.0-beta.24266.4">
+    <Dependency Name="Microsoft.DotNet.Arcade.Sdk" Version="6.0.0-beta.24326.2">
       <Uri>https://github.com/dotnet/arcade</Uri>
-      <Sha>2eab07c3d7b78219d10099b19fafeef2ecae1779</Sha>
+      <Sha>1844d819e6f33f1106083c5066ea01e0310eefa3</Sha>
       <SourceBuild RepoName="arcade" ManagedOnly="true" />
     </Dependency>
-    <Dependency Name="Microsoft.DotNet.Build.Tasks.Installers" Version="6.0.0-beta.24266.4">
+    <Dependency Name="Microsoft.DotNet.Build.Tasks.Installers" Version="6.0.0-beta.24326.2">
       <Uri>https://github.com/dotnet/arcade</Uri>
-      <Sha>2eab07c3d7b78219d10099b19fafeef2ecae1779</Sha>
+      <Sha>1844d819e6f33f1106083c5066ea01e0310eefa3</Sha>
     </Dependency>
-    <Dependency Name="Microsoft.DotNet.Build.Tasks.Templating" Version="6.0.0-beta.24266.4">
+    <Dependency Name="Microsoft.DotNet.Build.Tasks.Templating" Version="6.0.0-beta.24326.2">
       <Uri>https://github.com/dotnet/arcade</Uri>
-      <Sha>2eab07c3d7b78219d10099b19fafeef2ecae1779</Sha>
+      <Sha>1844d819e6f33f1106083c5066ea01e0310eefa3</Sha>
     </Dependency>
-    <Dependency Name="Microsoft.DotNet.Helix.Sdk" Version="6.0.0-beta.24266.4">
+    <Dependency Name="Microsoft.DotNet.Helix.Sdk" Version="6.0.0-beta.24326.2">
       <Uri>https://github.com/dotnet/arcade</Uri>
-      <Sha>2eab07c3d7b78219d10099b19fafeef2ecae1779</Sha>
+      <Sha>1844d819e6f33f1106083c5066ea01e0310eefa3</Sha>
     </Dependency>
   </ToolsetDependencies>
 </Dependencies>

eng/Versions.props

@@ -131,8 +131,8 @@
     <MicrosoftEntityFrameworkCoreVersion>6.0.30</MicrosoftEntityFrameworkCoreVersion>
     <MicrosoftEntityFrameworkCoreDesignVersion>6.0.30</MicrosoftEntityFrameworkCoreDesignVersion>
     <!-- Packages from dotnet/arcade -->
-    <MicrosoftDotNetBuildTasksInstallersVersion>6.0.0-beta.24266.4</MicrosoftDotNetBuildTasksInstallersVersion>
-    <MicrosoftDotNetBuildTasksTemplatingVersion>6.0.0-beta.24266.4</MicrosoftDotNetBuildTasksTemplatingVersion>
+    <MicrosoftDotNetBuildTasksInstallersVersion>6.0.0-beta.24326.2</MicrosoftDotNetBuildTasksInstallersVersion>
+    <MicrosoftDotNetBuildTasksTemplatingVersion>6.0.0-beta.24326.2</MicrosoftDotNetBuildTasksTemplatingVersion>
   </PropertyGroup>
   <!--
 

eng/common/templates-official/job/source-build.yml

@@ -31,6 +31,12 @@ parameters:
   #   container and pool.
   platform: {}
 
+  # If set to true and running on a non-public project,
+  # Internal blob storage locations will be enabled.
+  # This is not enabled by default because many repositories do not need internal sources
+  # and do not need to have the required service connections approved in the pipeline.
+  enableInternalSources: false
+
 jobs:
 - job: ${{ parameters.jobNamePrefix }}_${{ parameters.platform.name }}
   displayName: Source-Build (${{ parameters.platform.name }})
@@ -59,6 +65,8 @@ jobs:
     clean: all
 
   steps:
+  - ${{ if eq(parameters.enableInternalSources, true) }}:
+    - template: /eng/common/templates-official/steps/enable-internal-runtimes.yml
   - template: /eng/common/templates-official/steps/source-build.yml
     parameters:
       platform: ${{ parameters.platform }}

eng/common/templates-official/job/source-index-stage1.yml

@@ -1,6 +1,7 @@
 parameters:
   runAsPublic: false
-  sourceIndexPackageVersion: 1.0.1-20240320.1
+  sourceIndexUploadPackageVersion: 2.0.0-20240502.12
+  sourceIndexProcessBinlogPackageVersion: 1.0.1-20240129.2
   sourceIndexPackageSource: https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools/nuget/v3/index.json
   sourceIndexBuildCommand: powershell -NoLogo -NoProfile -ExecutionPolicy Bypass -Command "eng/common/build.ps1 -restore -build -binarylog -ci"
   preSteps: []
@@ -17,14 +18,14 @@ jobs:
   dependsOn: ${{ parameters.dependsOn }}
   condition: ${{ parameters.condition }}
   variables:
-  - name: SourceIndexPackageVersion
-    value: ${{ parameters.sourceIndexPackageVersion }}
+  - name: SourceIndexUploadPackageVersion
+    value: ${{ parameters.sourceIndexUploadPackageVersion }}
+  - name: SourceIndexProcessBinlogPackageVersion
+    value: ${{ parameters.sourceIndexProcessBinlogPackageVersion }}
   - name: SourceIndexPackageSource
     value: ${{ parameters.sourceIndexPackageSource }}
   - name: BinlogPath
     value: ${{ parameters.binlogPath }}
-  - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}:
-    - group: source-dot-net stage1 variables
 
   pool: ${{ parameters.pool }}
   steps:
@@ -40,8 +41,8 @@ jobs:
       workingDirectory: $(Agent.TempDirectory)
 
   - script: |
-      $(Agent.TempDirectory)/dotnet/dotnet tool install BinLogToSln --version $(SourceIndexPackageVersion) --add-source $(SourceIndexPackageSource) --tool-path $(Agent.TempDirectory)/.source-index/tools
-      $(Agent.TempDirectory)/dotnet/dotnet tool install UploadIndexStage1 --version $(SourceIndexPackageVersion) --add-source $(SourceIndexPackageSource) --tool-path $(Agent.TempDirectory)/.source-index/tools
+      $(Agent.TempDirectory)/dotnet/dotnet tool install BinLogToSln --version $(sourceIndexProcessBinlogPackageVersion) --add-source $(SourceIndexPackageSource) --tool-path $(Agent.TempDirectory)/.source-index/tools
+      $(Agent.TempDirectory)/dotnet/dotnet tool install UploadIndexStage1 --version $(sourceIndexUploadPackageVersion) --add-source $(SourceIndexPackageSource) --tool-path $(Agent.TempDirectory)/.source-index/tools
     displayName: Download Tools
     # Set working directory to temp directory so 'dotnet' doesn't try to use global.json and use the repo's sdk.
     workingDirectory: $(Agent.TempDirectory)
@@ -53,7 +54,21 @@ jobs:
     displayName: Process Binlog into indexable sln
 
   - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}:
-    - script: $(Agent.TempDirectory)/.source-index/tools/UploadIndexStage1 -i .source-index/stage1output -n $(Build.Repository.Name)
+    - task: AzureCLI@2
+      displayName: Get stage 1 auth token
+      inputs:
+        azureSubscription: 'SourceDotNet Stage1 Publish'
+        addSpnToEnvironment: true
+        scriptType: 'ps'
+        scriptLocation: 'inlineScript'
+        inlineScript: |
+          echo "##vso[task.setvariable variable=ARM_CLIENT_ID;issecret=true]$env:servicePrincipalId"
+          echo "##vso[task.setvariable variable=ARM_ID_TOKEN;issecret=true]$env:idToken"
+          echo "##vso[task.setvariable variable=ARM_TENANT_ID;issecret=true]$env:tenantId"
+
+    - script: |
+        az login --service-principal -u $(ARM_CLIENT_ID) --tenant $(ARM_TENANT_ID) --allow-no-subscriptions --federated-token $(ARM_ID_TOKEN)
+      displayName: "Login to Azure"
+
+    - script: $(Agent.TempDirectory)/.source-index/tools/UploadIndexStage1 -i .source-index/stage1output -n $(Build.Repository.Name) -s netsourceindexstage1 -b stage1
       displayName: Upload stage1 artifacts to source index
-      env:
-        BLOB_CONTAINER_URL: $(source-dot-net-stage1-blob-container-url)

eng/common/templates-official/jobs/source-build.yml

@@ -21,6 +21,12 @@ parameters:
   # one job runs on 'defaultManagedPlatform'.
   platforms: []
 
+  # If set to true and running on a non-public project,
+  # Internal nuget and blob storage locations will be enabled.
+  # This is not enabled by default because many repositories do not need internal sources
+  # and do not need to have the required service connections approved in the pipeline.
+  enableInternalSources: false
+
 jobs:
 
 - ${{ if ne(parameters.allCompletedJobId, '') }}:
@@ -38,9 +44,11 @@ jobs:
     parameters:
       jobNamePrefix: ${{ parameters.jobNamePrefix }}
       platform: ${{ platform }}
+      enableInternalSources: ${{ parameters.enableInternalSources }}
 
 - ${{ if eq(length(parameters.platforms), 0) }}:
   - template: /eng/common/templates-official/job/source-build.yml
     parameters:
       jobNamePrefix: ${{ parameters.jobNamePrefix }}
       platform: ${{ parameters.defaultManagedPlatform }}
+      enableInternalSources: ${{ parameters.enableInternalSources }}

eng/common/templates-official/post-build/common-variables.yml

@@ -2,7 +2,6 @@ variables:
   - group: AzureDevOps-Artifact-Feeds-Pats
   - group: DotNet-Blob-Feed
   - group: DotNet-DotNetCli-Storage
-  - group: DotNet-MSRC-Storage
   - group: Publish-Build-Assets
 
   # Whether the build is internal or not

eng/common/templates-official/steps/enable-internal-runtimes.yml

@@ -0,0 +1,28 @@
+# Obtains internal runtime download credentials and populates the 'dotnetbuilds-internal-container-read-token-base64'
+# variable with the base64-encoded SAS token, by default
+
+parameters:
+- name: federatedServiceConnection
+  type: string
+  default: 'dotnetbuilds-internal-read'
+- name: outputVariableName
+  type: string
+  default: 'dotnetbuilds-internal-container-read-token-base64'
+- name: expiryInHours
+  type: number
+  default: 1
+- name: base64Encode
+  type: boolean
+  default: true
+
+steps:
+- ${{ if ne(variables['System.TeamProject'], 'public') }}:
+  - template: /eng/common/templates-official/steps/get-delegation-sas.yml
+    parameters:
+      federatedServiceConnection: ${{ parameters.federatedServiceConnection }}
+      outputVariableName: ${{ parameters.outputVariableName }}
+      expiryInHours: ${{ parameters.expiryInHours }}
+      base64Encode: ${{ parameters.base64Encode }}
+      storageAccount: dotnetbuilds
+      container: internal
+      permissions: rl

eng/common/templates-official/steps/get-delegation-sas.yml

@@ -0,0 +1,43 @@
+parameters:
+- name: federatedServiceConnection
+  type: string
+- name: outputVariableName
+  type: string
+- name: expiryInHours
+  type: number
+  default: 1
+- name: base64Encode
+  type: boolean
+  default: false
+- name: storageAccount
+  type: string
+- name: container
+  type: string
+- name: permissions
+  type: string
+  default: 'rl'
+
+steps:
+- task: AzureCLI@2
+  displayName: 'Generate delegation SAS Token for ${{ parameters.storageAccount }}/${{ parameters.container }}'
+  inputs:
+    azureSubscription: ${{ parameters.federatedServiceConnection }}
+    scriptType: 'pscore'
+    scriptLocation: 'inlineScript'
+    inlineScript: |
+      # Calculate the expiration of the SAS token and convert to UTC
+      $expiry = (Get-Date).AddHours(${{ parameters.expiryInHours }}).ToUniversalTime().ToString("yyyy-MM-ddTHH:mm:ssZ")
+
+      $sas = az storage container generate-sas --account-name ${{ parameters.storageAccount }} --name ${{ parameters.container }} --permissions ${{ parameters.permissions }} --expiry $expiry --auth-mode login --as-user -o tsv
+
+      if ($LASTEXITCODE -ne 0) {
+        Write-Error "Failed to generate SAS token."
+        exit 1
+      }
+
+      if ('${{ parameters.base64Encode }}' -eq 'true') {
+        $sas = [Convert]::ToBase64String([System.Text.Encoding]::UTF8.GetBytes($sas))
+      }
+
+      Write-Host "Setting '${{ parameters.outputVariableName }}' with the access token value"
+      Write-Host "##vso[task.setvariable variable=${{ parameters.outputVariableName }};issecret=true]$sas"