bring updated files in from 'main'. need to swap to 1es-publish-artifact + the new archetype-tests-generate, but that's coming after merging main into 1es-template-conversion

scbedd · scbedd · commit f879d67adf5a · 2024-03-04T15:59:36.000-08:00
diff --git a/eng/common/pipelines/templates/jobs/archetype-sdk-tests-generate.yml b/eng/common/pipelines/templates/jobs/archetype-sdk-tests-generate.yml
@@ -31,9 +31,6 @@ parameters:
 - name: OsVmImage
   type: string
   default: ubuntu-22.04
-- name: Os
-  type: string
-  default: linux
 # This parameter is only necessary if there are multiple invocations of this template within the SAME STAGE.
 # When that occurs, provide a name other than the default value.
 - name: GenerateJobName
@@ -42,31 +39,15 @@ parameters:
 - name: PreGenerationSteps
   type: stepList
   default: []
-# Mappings to OS name required at template compile time by 1es pipeline templates
-- name: Pools
-  type: object
-  default:
-    - name: LinuxPool
-      os: linux
-    - name: LinuxNextPool
-      os: linux
-    - name: WindowsPool
-      os: windows
-    - name: MacPool
-      os: macOS
 
 jobs:
 - job: ${{ parameters.GenerateJobName }}
   variables:
-    - template: /eng/pipelines/templates/variables/image.yml
-    - name: skipComponentGovernanceDetection
-      value: true
-    - name: displayNameFilter
-      value: $[ coalesce(variables.jobMatrixFilter, '.*') ]
+    skipComponentGovernanceDetection: true
+    displayNameFilter: $[ coalesce(variables.jobMatrixFilter, '.*') ]
   pool:
     name: ${{ parameters.Pool }}
     vmImage: ${{ parameters.OsVmImage }}
-    os: ${{ parameters.Os }}
   ${{ if parameters.DependsOn }}:
     dependsOn: ${{ parameters.DependsOn }}
   steps:
@@ -84,56 +65,52 @@ jobs:
     - ${{ parameters.PreGenerationSteps }}
 
     - ${{ each config in parameters.MatrixConfigs }}:
-      - ${{ each pool in parameters.Pools }}:
-        - ${{ if eq(config.GenerateVMJobs, 'true') }}:
-          - task: Powershell@2
-            inputs:
-              pwsh: true
-              filePath: eng/common/scripts/job-matrix/Create-JobMatrix.ps1
-              arguments: >
-                -ConfigPath ${{ config.Path }}
-                -Selection ${{ config.Selection }}
-                -DisplayNameFilter '$(displayNameFilter)'
-                -Filters '${{ join(''',''', parameters.MatrixFilters) }}', 'container=^$', 'SupportedClouds=^$|${{ parameters.CloudConfig.Cloud }}', 'Pool=.*${{ pool.name }}$'
-                -Replace '${{ join(''',''', parameters.MatrixReplace) }}'
-                -NonSparseParameters '${{ join(''',''', config.NonSparseParameters) }}'
-            displayName: Create ${{ pool.name }} Matrix ${{ config.Name }}
-            name: vm_job_matrix_${{ config.Name }}_${{ pool.name }}
+      - ${{ if eq(config.GenerateVMJobs, 'true') }}:
+        - task: Powershell@2
+          inputs:
+            pwsh: true
+            filePath: eng/common/scripts/job-matrix/Create-JobMatrix.ps1
+            arguments: >
+              -ConfigPath ${{ config.Path }}
+              -Selection ${{ config.Selection }}
+              -DisplayNameFilter '$(displayNameFilter)'
+              -Filters '${{ join(''',''', parameters.MatrixFilters) }}','container=^$','SupportedClouds=^$|${{ parameters.CloudConfig.Cloud }}'
+              -Replace '${{ join(''',''', parameters.MatrixReplace) }}'
+              -NonSparseParameters '${{ join(''',''', config.NonSparseParameters) }}'
+          displayName: Generate VM Job Matrix ${{ config.Name }}
+          name: generate_vm_job_matrix_${{ config.Name }}
 
-        - ${{ if eq(config.GenerateContainerJobs, 'true') }}:
-          - task: Powershell@2
-            inputs:
-              pwsh: true
-              filePath: eng/common/scripts/job-matrix/Create-JobMatrix.ps1
-              arguments: >
-                -ConfigPath ${{ config.Path }}
-                -Selection ${{ config.Selection }}
-                -DisplayNameFilter '$(displayNameFilter)'
-                -Filters '${{ join(''',''', parameters.MatrixFilters) }}', 'container=^$', 'SupportedClouds=^$|${{ parameters.CloudConfig.Cloud }}', 'Pool=.*${{ pool.name }}$'
-                -NonSparseParameters '${{ join(''',''', config.NonSparseParameters) }}'
-            displayName: Create ${{ pool.name }} Container Matrix ${{ config.Name }}
-            name: container_job_matrix_${{ config.Name }}_${{ pool.name }}
+      - ${{ if eq(config.GenerateContainerJobs, 'true') }}:
+        - task: Powershell@2
+          inputs:
+            pwsh: true
+            filePath: eng/common/scripts/job-matrix/Create-JobMatrix.ps1
+            arguments: >
+              -ConfigPath ${{ config.Path }}
+              -Selection ${{ config.Selection }}
+              -DisplayNameFilter '$(displayNameFilter)'
+              -Filters '${{ join(''',''', parameters.MatrixFilters) }}', 'container=.*', 'SupportedClouds=^$|${{ parameters.CloudConfig.Cloud }}'
+              -NonSparseParameters '${{ join(''',''', config.NonSparseParameters) }}'
+          displayName: Generate Container Job Matrix
+          name: generate_container_job_matrix_${{ config.Name }}
 
 - ${{ each config in parameters.MatrixConfigs }}:
-  - ${{ each pool in parameters.Pools }}:
-    - ${{ if eq(config.GenerateVMJobs, 'true') }}:
-      - template: ${{ parameters.JobTemplatePath }}
-        parameters:
-          UsePlatformContainer: false
-          OSName: ${{ pool.os }}
-          Matrix: dependencies.${{ parameters.GenerateJobName }}.outputs['vm_job_matrix_${{ config.Name }}_${{ pool.name }}.matrix']
-          DependsOn: ${{ parameters.GenerateJobName }}
-          CloudConfig: ${{ parameters.CloudConfig }}
-          ${{ each param in parameters.AdditionalParameters }}:
-            ${{ param.key }}: ${{ param.value }}
+  - ${{ if eq(config.GenerateVMJobs, 'true') }}:
+    - template: ${{ parameters.JobTemplatePath }}
+      parameters:
+        UsePlatformContainer: false
+        Matrix: dependencies.${{ parameters.GenerateJobName }}.outputs['generate_vm_job_matrix_${{ config.Name }}.matrix']
+        DependsOn: ${{ parameters.GenerateJobName }}
+        CloudConfig: ${{ parameters.CloudConfig }}
+        ${{ each param in parameters.AdditionalParameters }}:
+          ${{ param.key }}: ${{ param.value }}
 
-    - ${{ if eq(config.GenerateContainerJobs, 'true') }}:
-      - template: ${{ parameters.JobTemplatePath }}
-        parameters:
-          UsePlatformContainer: true
-          OSName: ${{ pool.os }}
-          Matrix: dependencies.${{ parameters.GenerateJobName }}.outputs['vm_job_matrix_${{ config.Name }}_${{ pool.name }}.matrix']
-          DependsOn: ${{ parameters.GenerateJobName }}
-          CloudConfig: ${{ parameters.CloudConfig }}
-          ${{ each param in parameters.AdditionalParameters }}:
-            ${{ param.key }}: ${{ param.value }}
+  - ${{ if eq(config.GenerateContainerJobs, 'true') }}:
+    - template: ${{ parameters.JobTemplatePath }}
+      parameters:
+        UsePlatformContainer: true
+        Matrix: dependencies.${{ parameters.GenerateJobName }}.outputs['generate_container_job_matrix_${{ config.Name }}.matrix']
+        DependsOn: ${{ parameters.GenerateJobName }}
+        CloudConfig: ${{ parameters.CloudConfig }}
+        ${{ each param in parameters.AdditionalParameters }}:
+          ${{ param.key }}: ${{ param.value }}
diff --git a/eng/common/pipelines/templates/steps/publish-artifact.yml b/eng/common/pipelines/templates/steps/publish-artifact.yml
@@ -12,16 +12,16 @@ parameters:
   CustomCondition: true
 
 steps:
-  - task: 1ES.PublishPipelineArtifact@1
+  - task: PublishPipelineArtifact@1
     condition: and(succeeded(), ${{ parameters.CustomCondition }})
     displayName: 'Publish ${{ parameters.ArtifactName }} Artifacts'
     inputs:
-      artifact: '${{ parameters.ArtifactName }}'
+      artifactName: '${{ parameters.ArtifactName }}'
       path: '${{ parameters.ArtifactPath }}'
 
-  - task: 1ES.PublishPipelineArtifact@1
+  - task: PublishPipelineArtifact@1
     condition: and(failed(), ${{ parameters.CustomCondition }})
     displayName: 'Publish failed ${{ parameters.ArtifactName }} Artifacts'
     inputs:
-      artifact: '${{ parameters.ArtifactName }}-FailedAttempt$(System.JobAttempt)'
-      path: '${{ parameters.ArtifactPath }}'
+      artifactName: '${{ parameters.ArtifactName }}-FailedAttempt$(System.JobAttempt)'
+      path: '${{ parameters.ArtifactPath }}'
diff --git a/eng/common/scripts/Verify-AgentOS.ps1 b/eng/common/scripts/Verify-AgentOS.ps1
@@ -10,9 +10,9 @@ function Throw-InvalidOperatingSystem {
 
 if ($IsWindows -and $AgentImage -match "windows|win|MMS\d{4}") {
     $osName = "Windows"
-} elseif ($IsLinux -and $AgentImage -match "ubuntu|linux") {
+} elseif ($IsLinux -and $AgentImage -match "ubuntu") {
     $osName = "Linux"
-} elseif ($IsMacOs -and $AgentImage -match "macos|macOS") {
+} elseif ($IsMacOs -and $AgentImage -match "macos") {
     $osName = "macOS"
 } else {
     Throw-InvalidOperatingSystem
diff --git a/eng/common/scripts/job-matrix/job-matrix-functions.ps1 b/eng/common/scripts/job-matrix/job-matrix-functions.ps1
@@ -338,6 +338,9 @@ function ProcessReplace {
     $replaceMatrix = @()
 
     foreach ($element in $matrix) {
+        if (!$element -or $element.Count -eq 0) {
+            continue
+        }
         $replacement = [MatrixParameter[]]@()
         if (!$element -or $element.Count -eq 0) {
             continue
@@ -372,6 +375,7 @@ function ProcessReplace {
 
 function ProcessEnvironmentVariableReferences([array]$matrix, $displayNamesLookup) {
     $updatedMatrix = @()
+    $missingEnvVars = @{}
 
     foreach ($element in $matrix) {
         $updated = [MatrixParameter[]]@()
@@ -384,9 +388,9 @@ function ProcessEnvironmentVariableReferences([array]$matrix, $displayNamesLooku
             foreach ($flattened in $perm.Flatten()) {
                 if ($flattened.Value -is [string] -and $flattened.Value.StartsWith("env:")) {
                     $envKey = $flattened.Value.Replace("env:", "")
-                    $value = [System.Environment]::GetEnvironmentVariable($envKey) ?? ""
+                    $value = [System.Environment]::GetEnvironmentVariable($envKey)
                     if (!$value) {
-                        Write-Warning "Environment variable `"$envKey`" was not found or is empty."
+                        $missingEnvVars[$envKey] = $true
                     }
                     $perm.Set($value, $flattened.Name)
                 }
@@ -398,6 +402,9 @@ function ProcessEnvironmentVariableReferences([array]$matrix, $displayNamesLooku
         $updatedMatrix += CreateMatrixCombinationScalar $updated $displayNamesLookup
     }
 
+    if ($missingEnvVars.Count -gt 0) {
+        throw "Environment variables '$($missingEnvVars.Keys -join ", ")' were empty or not found."
+    }
     return $updatedMatrix
 }
 
