Generation pipeline migration to 1ES

Author: adrian05-ms

.azure-pipelines/generation-pipeline.yml

@@ -1,5 +1,5 @@
 steps:
-- template: use-dotnet-sdk.yml
+- template: /.azure-pipelines/generation-templates/use-dotnet-sdk.yml@self
   parameters:
     version: "9.x" #kiota uses a net9 target
 
@@ -15,9 +15,4 @@ steps:
     sourceFolder: '$(Build.SourcesDirectory)/src/kiota/bin/$(BuildConfiguration)/net9.0'
     contents: '**/*'
     targetFolder: '$(Build.ArtifactStagingDirectory)'
-  displayName: Copy Kiota executable
-
-- task: PublishBuildArtifacts@1
-  inputs:
-    pathToPublish: '$(Build.ArtifactStagingDirectory)'
-    artifactName: kiota
+  displayName: Copy Kiota executable

.azure-pipelines/generation-templates/build-and-publish-kiota.yml

@@ -1,6 +1,5 @@
 steps:
-- template: use-dotnet-sdk.yml
-
+- template: /.azure-pipelines/generation-templates/use-dotnet-sdk.yml@self
 - checkout: self
   displayName: checkout generator
   fetchDepth: 1
@@ -15,9 +14,4 @@ steps:
     sourceFolder: '$(Build.SourcesDirectory)/src/Typewriter/bin/$(BuildConfiguration)/net8.0'
     contents: '**/*'
     targetFolder: '$(Build.ArtifactStagingDirectory)'
-  displayName: Copy Typewriter executable
-
-- task: PublishBuildArtifacts@1
-  inputs:
-    pathToPublish: '$(Build.ArtifactStagingDirectory)'
-    artifactName: typewriter
+  displayName: Copy Typewriter executable

.azure-pipelines/generation-templates/build-and-publish-typewriter.yml

@@ -45,8 +45,8 @@ steps:
   fetchDepth: 1
   persistCredentials: true
 
-- template: checkout-metadata.yml
-- template: set-user-config.yml
+- template: /.azure-pipelines/generation-templates/checkout-metadata.yml@self
+- template: /.azure-pipelines/generation-templates/set-user-config.yml@self
 
 # required for TypeSpec
 - task: UseNode@1
@@ -76,14 +76,10 @@ steps:
     inputMetadataFile: ${{ parameters.inputMetadata }}
   enabled: true
 
-- template: download-typewriter.yml
-
 # required for typewriter
-- template: use-dotnet-sdk.yml
+- template: /.azure-pipelines/generation-templates/use-dotnet-sdk.yml@self
   parameters:
     version: '8.x'
-
-## Only run if the previous step was successful
 - pwsh: '$(scriptsDirectory)/run-typewriter-clean-metadata.ps1'
   env:
     OutputPath: ${{ parameters.outputPath }}
@@ -153,20 +149,11 @@ steps:
     contents: '**/*'
     targetFolder: '$(Build.ArtifactStagingDirectory)'
   displayName: Copy generated metadata
-
-- task: PublishBuildArtifacts@1
-  inputs:
-    pathToPublish: '$(Build.ArtifactStagingDirectory)'
-    artifactName: ${{ parameters.cleanMetadataFolder }}
-
-# Use the clean metadata from the last step to generate DotNet files.
-
-# required for the hidi installation validation
-- template: use-dotnet-sdk.yml
+- template: /.azure-pipelines/generation-templates/use-dotnet-sdk.yml@self
   parameters:
     version: '9.x'
 
-- pwsh: dotnet tool install --global Microsoft.OpenApi.Hidi --version 1.6.24 
+- pwsh: dotnet tool install --global Microsoft.OpenApi.Hidi --version 1.6.24
   displayName: 'Install hidi tool'
 
 # verify that generated metadata is parsable as an Edm model

.azure-pipelines/generation-templates/capture-metadata.yml

@@ -1,9 +1,3 @@
-# Copyright (c) Microsoft Corporation.  All Rights Reserved.  Licensed under the MIT License.  See License in the project root for license information.
-# contains an end to end validation pipeline using C# compilation tests for staging beta metadata
-
-# capture-metadata.yml will result in the capture and cleaning of the metadata
-# with the artifacts posted to GitHub.
-
 parameters:
 - name: 'endpoint'
   type: string
@@ -13,7 +7,6 @@ parameters:
   default: $(System.ArtifactsDirectory)
 - name: 'cleanMetadataFolder'
   type: string
-
 jobs:
 - job: get_conversion_settings
   displayName: "Get conversion settings"
@@ -23,8 +16,7 @@ jobs:
     fetchDepth: 1
     persistCredentials: true
 
-  - template: checkout-metadata.yml
-
+  - template: /.azure-pipelines/generation-templates/checkout-metadata.yml@self
   - pwsh: |
       $dirPath = "./conversion-settings"
       $endpoint = "${{ parameters.endpoint }}"
@@ -41,38 +33,39 @@ jobs:
       Write-Host "##vso[task.setvariable variable=targets;isOutput=true]$json"
     name: setTargets
     workingDirectory: $(Build.SourcesDirectory)/msgraph-metadata
-
   - script: echo $(setTargets.targets)
     displayName: "Print settings"
-
 - job: convert_openapi
   dependsOn: get_conversion_settings
   displayName: Convert
   strategy:
     matrix: $[ dependencies.get_conversion_settings.outputs['setTargets.targets'] ]
   variables:
     targets: $[ dependencies.get_conversion_settings.outputs['setTargets.targets'] ]
-
+  templateContext:
+    outputs:
+    - output: pipelineArtifact
+      targetPath: '$(Build.ArtifactStagingDirectory)'
+      artifactName: ${{ parameters.cleanMetadataFolder }}
   steps:
-  # We only need the scripts
   - checkout: self
     displayName: checkout generator
     fetchDepth: 1
     persistCredentials: true
 
-  - template: checkout-metadata.yml
+  - template: /.azure-pipelines/generation-templates/checkout-metadata.yml@self
 
   # required for the hidi to run
-  - template: use-dotnet-sdk.yml
+  - template: /.azure-pipelines/generation-templates/use-dotnet-sdk.yml@self
     parameters:
       version: '8.x'
 
-  # required for the hidi installation validation
-  - template: use-dotnet-sdk.yml
+# required for the hidi installation validation
+  - template: /.azure-pipelines/generation-templates/use-dotnet-sdk.yml@self
     parameters:
       version: '9.x'
 
-  - pwsh: dotnet tool install --global Microsoft.OpenApi.Hidi --version 1.6.24 
+  - pwsh: dotnet tool install --global Microsoft.OpenApi.Hidi --version 1.6.24
     displayName: install hidi
 
   - pwsh: |
@@ -92,12 +85,6 @@ jobs:
       contents: '**/$(Name).yaml'
       targetFolder: '$(Build.ArtifactStagingDirectory)'
     displayName: Copy generated metadata
-
-  - task: PublishBuildArtifacts@1
-    inputs:
-      pathToPublish: '$(Build.ArtifactStagingDirectory)'
-      artifactName: ${{ parameters.cleanMetadataFolder }}
-
   - pwsh: |
       ./scripts/run-openapi-validation.ps1 -repoDirectory (Get-Location).Path -version "${{ parameters.endpoint }}" -platformName "$(Name)"
     displayName: ensure that OpenAPI docs can be parsed
@@ -107,21 +94,19 @@ jobs:
 - job: publish_openapi
   dependsOn: convert_openapi
   displayName: Publish
+  templateContext:
+    inputs:
+    - input: pipelineArtifact
+      targetPath: $(Build.SourcesDirectory)/msgraph-metadata/openapi/${{ parameters.endpoint }}
+      artifactName: ${{ parameters.cleanMetadataFolder }}
   steps:
   # We only need the scripts
   - checkout: self
     displayName: checkout generator
     fetchDepth: 1
     persistCredentials: true
-
-  - template: checkout-metadata.yml
-  - template: set-user-config.yml
-  - task: DownloadPipelineArtifact@2
-    inputs:
-      path: $(Build.SourcesDirectory)/msgraph-metadata/openapi/${{ parameters.endpoint }}
-      artifact: ${{ parameters.cleanMetadataFolder }}
-
-  # Checkin clean metadata into metadata repo or make it an artifact.
+  - template: /.azure-pipelines/generation-templates/checkout-metadata.yml@self
+  - template: /.azure-pipelines/generation-templates/set-user-config.yml@self
   - pwsh: '$(scriptsDirectory)/git-push-cleanmetadata.ps1'
 
     displayName: push clean ${{ parameters.endpoint }} OpenAPI description to msgraph-metadata repo

.azure-pipelines/generation-templates/capture-openapi.yml

@@ -1,9 +1,5 @@
+# This Yaml Document has been converted by ESAI Yaml Pipeline Conversion Tool.
+# Please make sure to check all the converted content, it is your team's responsibility to make sure that the pipeline is still valid and functions as expected.
+# The 'DownloadBuildArtifacts@0' tasks have been converted to inputs within the `templateContext` section of each job.
 steps:
-- task: DownloadBuildArtifacts@0
-  inputs:
-    buildType: 'current'
-    downloadType: 'single'
-    artifactName: 'kiota'
-    downloadPath: '$(Build.SourcesDirectory)'
-
-- bash: chmod +x $(Build.SourcesDirectory)/kiota/kiota
+- bash: chmod +x $(Build.SourcesDirectory)/kiota/kiota

.azure-pipelines/generation-templates/download-kiota.yml

@@ -1,7 +1,4 @@
-steps:
-- task: DownloadBuildArtifacts@0
-  inputs:
-    buildType: 'current'
-    downloadType: 'single'
-    artifactName: 'typewriter'
-    downloadPath: '$(Build.SourcesDirectory)'
+# This Yaml Document has been converted by ESAI Yaml Pipeline Conversion Tool.
+# Please make sure to check all the converted content, it is your team's responsibility to make sure that the pipeline is still valid and functions as expected.
+# The 'DownloadBuildArtifacts@0' tasks have been converted to inputs within the `templateContext` section of each job.
+steps: []

.azure-pipelines/generation-templates/download-typewriter.yml

@@ -57,11 +57,9 @@ parameters:
   default: "-e '/copilot' -e '/copilot/**'"
 
 steps:
-- template: set-up-for-generation-kiota.yml
+- template: /.azure-pipelines/generation-templates/set-up-for-generation-kiota.yml@self
   parameters:
     cleanMetadataFolder: ${{ parameters.cleanMetadataFolder }}
-    downloadSteps:
-    - template: download-kiota.yml
 
 - checkout: ${{ parameters.repoName }}
   displayName: 'checkout ${{ parameters.repoName }}'

.azure-pipelines/generation-templates/language-generation-kiota.yml

@@ -34,12 +34,9 @@ parameters:
   displayName: Generate PR after publish changes?
 
 steps:
-- template: set-up-for-generation.yml
+- template: /.azure-pipelines/generation-templates/set-up-for-generation.yml@self
   parameters:
     cleanMetadataFolder: ${{ parameters.cleanMetadataFolder }}
-    downloadSteps:
-    - template: download-typewriter.yml
-
 - checkout: ${{ parameters.repoName }}
   displayName: 'checkout ${{ parameters.repoName }}'
   fetchDepth: 1
@@ -67,7 +64,7 @@ steps:
     TypewriterExecutable: $(typewriterExecutable)
     TypewriterDirectory: $(typewriterDirectory)
     Language: ${{ parameters.language }}
-    Endpoint:  ${{ parameters.version }}
+    Endpoint: ${{ parameters.version }}
   displayName: 'Run Typewriter for ${{ parameters.language }} ${{ parameters.version }}'
 
 - ${{ parameters.languageSpecificSteps }}

.azure-pipelines/generation-templates/language-generation.yml

@@ -1,20 +1,13 @@
 parameters:
-  - name: cleanMetadataFolder
-    type: string
-  - name: downloadSteps
-    type: stepList
-    default:
-    - pwsh: |
-        Write-Error "No download step is provided"
+- name: cleanMetadataFolder
+  type: string
 
 steps:
-- template: set-user-config.yml
-- template: use-dotnet-sdk.yml
+- template: /.azure-pipelines/generation-templates/set-user-config.yml@self
+- template: /.azure-pipelines/generation-templates/use-dotnet-sdk.yml@self
   parameters:
     version: "9.x" #kiota uses a net9 target
 
-- ${{ parameters.downloadSteps }}
-
 # checkout metadata repo if capture and clean step is skipped
 - checkout: msgraph-metadata
   displayName: checkout metadata
@@ -30,16 +23,3 @@ steps:
     Move-Item -Path $(Build.SourcesDirectory)/msgraph-metadata/openapi/beta/openapi.yaml -Destination $(Build.SourcesDirectory)/msgraph-metadata/clean_beta_openapi/
   displayName: Move metadata to expected location
   condition: or(eq(variables.skipMetadataCaptureAndClean, true), eq(variables.skipOpenApiCaptureAndClean, true))
-
-# if capture and clean step is not skipped
-# then download the artifact from capture and clean steps
-# follow the same folder structure as msgraph-metadata repo
-# so that metadata reference path is always the same
-- task: DownloadBuildArtifacts@0
-  inputs:
-    buildType: 'current'
-    downloadType: 'single'
-    artifactName: ${{ parameters.cleanMetadataFolder }}
-    downloadPath: '$(Build.SourcesDirectory)/msgraph-metadata'
-  condition: and(eq(variables.skipMetadataCaptureAndClean, false), eq(variables.skipOpenApiCaptureAndClean, false))
-  displayName: Downloading metadata from artifacts