Sync eng/common directory with azure-sdk-tools repository (Azure#1867)

Author: azure-sdk

eng/common/TestResources/New-TestResources.ps1

@@ -350,6 +350,10 @@ try {
         # to determine whether resources should be removed.
         Write-Host "Setting variable 'CI_HAS_DEPLOYED_RESOURCES': 'true'"
         LogVsoCommand "##vso[task.setvariable variable=CI_HAS_DEPLOYED_RESOURCES;]true"
+        # Set resource group env variable early in cases where deployment fails as we
+        # still want to clean up the group. The Remove-TestResources.ps1 script consumes this var.
+        $envVarName = (BuildServiceDirectoryPrefix $serviceName) + "RESOURCE_GROUP"
+        LogVsoCommand "##vso[task.setvariable variable=$envVarName;]$ResourceGroupName"
     }
 
     Log "Creating resource group '$ResourceGroupName' in location '$Location'"
@@ -515,9 +519,9 @@ try {
 
     # Try to detect the shell based on the parent process name (e.g. launch via shebang).
     $shell, $shellExportFormat = if (($parentProcessName = (Get-Process -Id $PID).Parent.ProcessName) -and $parentProcessName -eq 'cmd') {
-        'cmd', 'set {0}={1}'
+        'cmd', 'set {0}=''{1}'''
     } elseif (@('bash', 'csh', 'tcsh', 'zsh') -contains $parentProcessName) {
-        'shell', 'export {0}={1}'
+        'shell', 'export {0}=''{1}'''
     } else {
         'PowerShell', '${{env:{0}}} = ''{1}'''
     }

eng/common/TestResources/SubConfig-Helpers.ps1

@@ -196,17 +196,26 @@ function UpdateSubscriptionConfigurationWithFiles([object]$baseSubConfig, [strin
 # Helper function for processing stringified json sub configs from pipeline parameter data
 function BuildAndSetSubscriptionConfig([string]$baseSubConfigJson, [string]$additionalSubConfigsJson, [string]$subConfigFilesJson) {
   $finalConfig = @{}
-  if ($baseSubConfigJson) {
+
+  if ($baseSubConfigJson -and $baseSubConfigJson -ne '""') {
+    # When variable groups are not added to the pipeline, secret references like
+    # $(<my secret>) are passed as a string literal instead of being replaced by the keyvault secret value
+    if ($baseSubConfigJson -notlike '{*') {
+      throw "Expected a json dictionary object but found '$baseSubConfigJson'. This probably means a subscription config secret was not downloaded. The pipeline is likely missing a variable group."
+    }
     $baseSubConfig = $baseSubConfigJson | ConvertFrom-Json -AsHashtable
 
     Write-Host "Setting base sub config"
     $finalConfig = SetSubscriptionConfiguration $baseSubConfig
   }
 
-  if ($additionalSubConfigsJson) {
+  if ($additionalSubConfigsJson -and $additionalSubConfigsJson -ne '""') {
     $subConfigs = $additionalSubConfigsJson | ConvertFrom-Json -AsHashtable
 
     foreach ($subConfig in $subConfigs) {
+      if ($subConfig -isnot [hashtable]) {
+        throw "Expected a json dictionary object but found '$subConfig'. This probably means a subscription config secret was not downloaded. The pipeline is likely missing a variable group."
+      }
       Write-Host "Merging sub config from list"
       $finalConfig = UpdateSubscriptionConfiguration $finalConfig $subConfig
     }

eng/common/TestResources/build-test-resource-config.yml

@@ -1,7 +1,7 @@
 parameters:
   - name: SubscriptionConfiguration
     type: string
-    default: $(sub-config-azure-cloud-test-resources)
+    default: '{}'
   - name: SubscriptionConfigurations
     type: object
     default: null

eng/common/TestResources/deploy-test-resources.yml

@@ -4,10 +4,11 @@ parameters:
   DeleteAfterHours: 8
   Location: ''
   EnvVars: {}
-  SubscriptionConfiguration: $(sub-config-azure-cloud-test-resources)
+  SubscriptionConfiguration: '{}'
   ServiceConnection: not-specified
   ResourceType: test
-  UseFederatedAuth: false
+  UseFederatedAuth: true
+  PersistOidcToken: false
 
 # SubscriptionConfiguration will be splatted into the parameters of the test
 # resources script. It should be JSON in the form:
@@ -35,18 +36,32 @@ parameters:
 #   }
 # }
 
-
 steps:
   - template: /eng/common/pipelines/templates/steps/cache-ps-modules.yml
 
   - template: /eng/common/TestResources/setup-environments.yml
 
+  - ${{ if eq(parameters.PersistOidcToken, true) }}:
+    - task: AzureCLI@2
+      displayName: Set OIDC token
+      env:
+        ARM_OIDC_TOKEN: $(ARM_OIDC_TOKEN)
+      inputs:
+        azureSubscription: ${{ parameters.ServiceConnection }}
+        addSpnToEnvironment: true
+        scriptLocation: inlineScript
+        scriptType: pscore
+        inlineScript: |
+          Write-Host "##vso[task.setvariable variable=ARM_OIDC_TOKEN;issecret=true]$($env:idToken)"
+
   - ${{ if eq('true', parameters.UseFederatedAuth) }}:
     - task: AzurePowerShell@5
-      displayName: Deploy test resources
+      displayName: 🚀 Deploy test resources
       env:
         TEMP: $(Agent.TempDirectory)
         PoolSubnet: $(PoolSubnet)
+        ${{ if eq(parameters.PersistOidcToken, true) }}:
+          ARM_OIDC_TOKEN: $(ARM_OIDC_TOKEN)
         ${{ insert }}: ${{ parameters.EnvVars }}
       inputs:
         azureSubscription: ${{ parameters.ServiceConnection }}
@@ -59,6 +74,21 @@ steps:
             ${{ parameters.SubscriptionConfiguration }}
           '@ | ConvertFrom-Json -AsHashtable;
 
+          $context = Get-AzContext
+          $subscriptionConfiguration["Environment"] = $context.Environment.Name
+          $subscriptionConfiguration["SubscriptionId"] = $context.Subscription.Id
+          $subscriptionConfiguration["TenantId"] = $context.Subscription.TenantId
+          $subscriptionConfiguration["TestApplicationId"] = $context.Account.Id
+          $subscriptionConfiguration["ProvisionerApplicationId"] = $context.Account.Id
+
+          $principal = Get-AzADServicePrincipal -ApplicationId $context.Account.Id
+          $subscriptionConfiguration["TestApplicationOid"] = $principal.Id
+          $subscriptionConfiguration["ProvisionerApplicationOid"] = $principal.Id
+
+          Write-Host ($subscriptionConfiguration | ConvertTo-Json)
+          # Write the new SubscriptionConfiguration to be used by the remove test resources
+          Write-Host "##vso[task.setvariable variable=SubscriptionConfiguration;]$($subscriptionConfiguration | ConvertTo-Json -Compress)"
+
           # The subscriptionConfiguration may have ArmTemplateParameters defined, so
           # pass those in via the ArmTemplateParameters flag, and handle any
           # additional parameters from the pipelines via AdditionalParameters
@@ -96,7 +126,7 @@ steps:
           -ServicePrincipalAuth `
           -Force `
           -Verbose | Out-Null
-      displayName: Deploy test resources
+      displayName: 🚀 Deploy test resources
       env:
         TEMP: $(Agent.TempDirectory)
         PoolSubnet: $(PoolSubnet)

eng/common/TestResources/remove-test-resources.yml

@@ -3,11 +3,11 @@
 
 parameters:
   ServiceDirectory: ''
-  SubscriptionConfiguration: $(sub-config-azure-cloud-test-resources)
+  SubscriptionConfiguration: $(SubscriptionConfiguration)
   ServiceConnection: not-specified
   ResourceType: test
   EnvVars: {}
-  UseFederatedAuth: false
+  UseFederatedAuth: true
 
 # SubscriptionConfiguration will be splat into the parameters of the test
 # resources script. It should be JSON in the form:

eng/common/TestResources/sub-config/AzureChinaMsft.json

@@ -31,7 +31,7 @@ stages:
       vmImage: ubuntu-22.04
 
     variables:
-      CodeownersLinterVersion: '1.0.0-dev.20240614.4'
+      CodeownersLinterVersion: '1.0.0-dev.20240926.2'
       DotNetDevOpsFeed: "https://pkgs.dev.azure.com/azure-sdk/public/_packaging/azure-sdk-for-net/nuget/v3/index.json"
       RepoLabelUri: "https://azuresdkartifacts.blob.core.windows.net/azure-sdk-write-teams/repository-labels-blob"
       TeamUserUri: "https://azuresdkartifacts.blob.core.windows.net/azure-sdk-write-teams/azure-sdk-write-teams-blob"