Sync eng/common directory with azure-sdk-tools for PR 8760 (#36732)

* Do not mark values as secret that are from git-hosted sub config files

* Simplify sub config build yaml

---------

Co-authored-by: Ben Broderick Phillips <[email protected]>

Author: azure-sdk

eng/common/TestResources/SubConfig-Helpers.ps1

@@ -82,7 +82,8 @@ function ShouldMarkValueAsSecret([string]$serviceName, [string]$key, [string]$va
         "TenantId",
         "TestApplicationId",
         "TestApplicationOid",
-        "ProvisionerApplicationId"
+        "ProvisionerApplicationId",
+        "ProvisionerApplicationOid"
     )
 
     $serviceDirectoryPrefix = BuildServiceDirectoryPrefix $serviceName
@@ -129,7 +130,7 @@ function SetSubscriptionConfiguration([object]$subscriptionConfiguration)
     return $subscriptionConfiguration
 }
 
-function UpdateSubscriptionConfiguration([object]$subscriptionConfigurationBase, [object]$subscriptionConfiguration)
+function UpdateSubscriptionConfiguration([object]$subscriptionConfigurationBase, [object]$subscriptionConfiguration, [array]$allowedValues)
 {
     foreach ($pair in $subscriptionConfiguration.GetEnumerator()) {
         if ($pair.Value -is [Hashtable]) {
@@ -140,13 +141,13 @@ function UpdateSubscriptionConfiguration([object]$subscriptionConfigurationBase,
                 # Mark values as secret so we don't print json blobs containing secrets in the logs.
                 # Prepend underscore to the variable name, so we can still access the variable names via environment
                 # variables if they get set subsequently.
-                if (ShouldMarkValueAsSecret "AZURE_" $nestedPair.Name $nestedPair.Value) {
+                if (ShouldMarkValueAsSecret "AZURE_" $nestedPair.Name $nestedPair.Value $allowedValues) {
                     Write-Host "##vso[task.setvariable variable=_$($nestedPair.Name);issecret=true;]$($nestedPair.Value)"
                 }
                 $subscriptionConfigurationBase[$pair.Name][$nestedPair.Name] = $nestedPair.Value
             }
         } else {
-            if (ShouldMarkValueAsSecret "AZURE_" $pair.Name $pair.Value) {
+            if (ShouldMarkValueAsSecret "AZURE_" $pair.Name $pair.Value $allowedValues) {
                 Write-Host "##vso[task.setvariable variable=_$($pair.Name);issecret=true;]$($pair.Value)"
             }
             $subscriptionConfigurationBase[$pair.Name] = $pair.Value
@@ -155,3 +156,66 @@ function UpdateSubscriptionConfiguration([object]$subscriptionConfigurationBase,
 
     return $subscriptionConfigurationBase
 }
+
+# Helper function for processing sub config files from a pipeline file list yaml parameter
+function UpdateSubscriptionConfigurationWithFiles([object]$baseSubConfig, [string]$fileListJson) {
+  if (!$fileListJson) {
+    return $baseSubConfig
+  }
+
+  $finalConfig = $baseSubConfig
+
+  $subConfigFiles = $fileListJson | ConvertFrom-Json -AsHashtable
+  foreach ($file in $subConfigFiles) {
+    # In some cases, $file could be an empty string. Get-Content will fail
+    # if $file is an empty string, so skip those cases.
+    if (!$file) {
+      continue
+    }
+
+    Write-Host "Merging sub config from file: $file"
+    $subConfig = Get-Content $file | ConvertFrom-Json -AsHashtable
+    $allowedValues = @()
+    # Since the keys are all coming from a file in github, we know every key should not be marked
+    # as a secret. Set up these exclusions here to make pipeline log debugging easier.
+    foreach ($pair in $subConfig.GetEnumerator()) {
+      if ($pair.Value -is [Hashtable]) {
+        foreach($nestedPair in $pair.Value.GetEnumerator()) {
+          $allowedValues += $nestedPair.Name
+        }
+      } else {
+        $allowedValues += $pair.Name
+      }
+    }
+    $finalConfig = UpdateSubscriptionConfiguration $finalConfig $subConfig $allowedValues
+  }
+
+  return $finalConfig
+}
+
+# Helper function for processing stringified json sub configs from pipeline parameter data
+function BuildAndSetSubscriptionConfig([string]$baseSubConfigJson, [string]$additionalSubConfigsJson, [string]$subConfigFilesJson) {
+  $finalConfig = @{}
+  if ($baseSubConfigJson) {
+    $baseSubConfig = $baseSubConfigJson | ConvertFrom-Json -AsHashtable
+
+    Write-Host "Setting base sub config"
+    $finalConfig = SetSubscriptionConfiguration $baseSubConfig
+  }
+
+  if ($additionalSubConfigsJson) {
+    $subConfigs = $additionalSubConfigsJson | ConvertFrom-Json -AsHashtable
+
+    foreach ($subConfig in $subConfigs) {
+      Write-Host "Merging sub config from list"
+      $finalConfig = UpdateSubscriptionConfiguration $finalConfig $subConfig
+    }
+  }
+
+  Write-Host "Merging sub config from files"
+  $finalConfig = UpdateSubscriptionConfigurationWithFiles $finalConfig $subConfigFilesJson
+
+  Write-Host ($finalConfig | ConvertTo-Json)
+  $serialized = $finalConfig | ConvertTo-Json -Compress
+  Write-Host "##vso[task.setvariable variable=SubscriptionConfiguration;]$serialized"
+}

eng/common/TestResources/build-test-resource-config.yml

@@ -37,50 +37,17 @@ steps:
   - pwsh: |
       . ./eng/common/TestResources/SubConfig-Helpers.ps1
 
-      $finalConfig = @{}
-      $baseSubConfigRaw = @'
+      $baseSubConfigJson = @'
         ${{ parameters.SubscriptionConfiguration }}
       '@.Trim()
-      if ($baseSubConfigRaw) {
-        $baseSubConfig = $baseSubConfigRaw | ConvertFrom-Json -AsHashtable
 
-        Write-Host "Setting base sub config"
-        $finalConfig = SetSubscriptionConfiguration $baseSubConfig
-      }
-
-      $subConfigJsonsRaw = @'
+      $additionalSubConfigsJson = @'
         ${{ convertToJson(parameters.SubscriptionConfigurations) }}
       '@.Trim() -replace '"{', '{' -replace '}"', '}'
 
-      if ($subConfigJsonsRaw) {
-        $subConfigs = $subConfigJsonsRaw | ConvertFrom-Json -AsHashtable
-
-        foreach ($subConfig in $subConfigs) {
-          Write-Host "Merging sub config from list"
-          $finalConfig = UpdateSubscriptionConfiguration $finalConfig $subConfig
-        }
-      }
-
-      $subConfigFilesRaw = @'
+      $subConfigFilesJson = @'
         ${{ convertToJson(parameters.SubscriptionConfigurationFilePaths) }}
       '@.Trim()
 
-      if ($subConfigFilesRaw) {
-        $subConfigFiles = $subConfigFilesRaw | ConvertFrom-Json -AsHashtable
-        foreach ($file in $subConfigFiles) {
-          # In some cases, $file could be an empty string. Get-Content will fail
-          # if $file is an empty string, so skip those cases.
-          if (!$file) {
-            continue
-          }
-
-          Write-Host "Merging sub config from file: $file"
-          $subConfig = Get-Content $file | ConvertFrom-Json -AsHashtable
-          $finalConfig = UpdateSubscriptionConfiguration $finalConfig $subConfig
-        }
-      }
-
-      Write-Host ($finalConfig | ConvertTo-Json)
-      $serialized = $finalConfig | ConvertTo-Json -Compress
-      Write-Host "##vso[task.setvariable variable=SubscriptionConfiguration;]$serialized"
+      BuildAndSetSubscriptionConfig $baseSubConfigJson $additionalSubConfigsJson $subConfigFilesJson
     displayName: Merge subscription configurations