Merge dev into main (#1380)

* Generation pipeline migration to 1ES

* fixing param issue

* adding missing repo

* Fix template issues

* duplication error

* removing download steps

* sdl analysis

* excluding repos

* changing typewriter targetPath

* changing download path

* adding typewriter to the target path

* Fixing openAPI outputPath

* fixing output path

* test run

* changing target path

* testing change

* multiple artifacts

* testing copy files

* hardcoding downloaded artifacts

* testing copy files

* Fixing copy files

* Attempting to fix kiota dir issue

* Testing downloaded paths

* moving from bash to powershell

* changing targetPath

* adding template context

* adding kiota repo to scan

* reverting back change

* reverting back copyright comments

* adding comments

* adding comment space

* chore: update Python copilot agent library generation dir (#1379)

* chore(deps): bump submodules/vipr from `c5c2204` to `39c1ee7` (#1378)

Bumps [submodules/vipr](https://github.com/microsoft/vipr) from `c5c2204` to `39c1ee7`.
- [Release notes](https://github.com/microsoft/vipr/releases)
- [Commits](microsoft/Vipr@c5c2204...39c1ee7)

---
updated-dependencies:
- dependency-name: submodules/vipr
  dependency-version: 39c1ee7affbd5567511890ddff5edb6c721ee1c7
  dependency-type: direct:production
...

Signed-off-by: dependabot[bot] <[email protected]>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
Co-authored-by: Michael Mainer <[email protected]>

* chore(deps): bump actions/checkout from 4 to 5 (#1376)

Bumps [actions/checkout](https://github.com/actions/checkout) from 4 to 5.
- [Release notes](https://github.com/actions/checkout/releases)
- [Changelog](https://github.com/actions/checkout/blob/main/CHANGELOG.md)
- [Commits](actions/checkout@v4...v5)

---
updated-dependencies:
- dependency-name: actions/checkout
  dependency-version: '5'
  dependency-type: direct:production
  update-type: version-update:semver-major
...

Signed-off-by: dependabot[bot] <[email protected]>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>

* chore(deps): bump actions/setup-dotnet from 4.3.1 to 5.0.0 (#1381)

Bumps [actions/setup-dotnet](https://github.com/actions/setup-dotnet) from 4.3.1 to 5.0.0.
- [Release notes](https://github.com/actions/setup-dotnet/releases)
- [Commits](actions/setup-dotnet@v4.3.1...v5.0.0)

---
updated-dependencies:
- dependency-name: actions/setup-dotnet
  dependency-version: 5.0.0
  dependency-type: direct:production
  update-type: version-update:semver-major
...

Signed-off-by: dependabot[bot] <[email protected]>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>

* dep: update vipr dependency (#1383)

Co-authored-by: Adrian <[email protected]>

* chore: fix path to beta python ccs (#1382)

* ci: update script to special case for beta typing versioning (#1384)

---------

Signed-off-by: dependabot[bot] <[email protected]>
Co-authored-by: Adrian <[email protected]>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>

Author: 3 people

.azure-pipelines/generation-pipeline.yml

@@ -1,5 +1,5 @@
 steps:
-- template: use-dotnet-sdk.yml
+- template: /.azure-pipelines/generation-templates/use-dotnet-sdk.yml@self
   parameters:
     version: "9.x" #kiota uses a net9 target
 
@@ -15,9 +15,4 @@ steps:
     sourceFolder: '$(Build.SourcesDirectory)/src/kiota/bin/$(BuildConfiguration)/net9.0'
     contents: '**/*'
     targetFolder: '$(Build.ArtifactStagingDirectory)'
-  displayName: Copy Kiota executable
-
-- task: PublishBuildArtifacts@1
-  inputs:
-    pathToPublish: '$(Build.ArtifactStagingDirectory)'
-    artifactName: kiota
+  displayName: Copy Kiota executable

.azure-pipelines/generation-templates/build-and-publish-kiota.yml

@@ -1,6 +1,5 @@
 steps:
-- template: use-dotnet-sdk.yml
-
+- template: /.azure-pipelines/generation-templates/use-dotnet-sdk.yml@self
 - checkout: self
   displayName: checkout generator
   fetchDepth: 1
@@ -15,9 +14,4 @@ steps:
     sourceFolder: '$(Build.SourcesDirectory)/src/Typewriter/bin/$(BuildConfiguration)/net8.0'
     contents: '**/*'
     targetFolder: '$(Build.ArtifactStagingDirectory)'
-  displayName: Copy Typewriter executable
-
-- task: PublishBuildArtifacts@1
-  inputs:
-    pathToPublish: '$(Build.ArtifactStagingDirectory)'
-    artifactName: typewriter
+  displayName: Copy Typewriter executable

.azure-pipelines/generation-templates/build-and-publish-typewriter.yml

@@ -45,8 +45,8 @@ steps:
   fetchDepth: 1
   persistCredentials: true
 
-- template: checkout-metadata.yml
-- template: set-user-config.yml
+- template: /.azure-pipelines/generation-templates/checkout-metadata.yml@self
+- template: /.azure-pipelines/generation-templates/set-user-config.yml@self
 
 # required for TypeSpec
 - task: UseNode@1
@@ -76,14 +76,10 @@ steps:
     inputMetadataFile: ${{ parameters.inputMetadata }}
   enabled: true
 
-- template: download-typewriter.yml
-
 # required for typewriter
-- template: use-dotnet-sdk.yml
+- template: /.azure-pipelines/generation-templates/use-dotnet-sdk.yml@self
   parameters:
     version: '8.x'
-
-## Only run if the previous step was successful
 - pwsh: '$(scriptsDirectory)/run-typewriter-clean-metadata.ps1'
   env:
     OutputPath: ${{ parameters.outputPath }}
@@ -153,20 +149,11 @@ steps:
     contents: '**/*'
     targetFolder: '$(Build.ArtifactStagingDirectory)'
   displayName: Copy generated metadata
-
-- task: PublishBuildArtifacts@1
-  inputs:
-    pathToPublish: '$(Build.ArtifactStagingDirectory)'
-    artifactName: ${{ parameters.cleanMetadataFolder }}
-
-# Use the clean metadata from the last step to generate DotNet files.
-
-# required for the hidi installation validation
-- template: use-dotnet-sdk.yml
+- template: /.azure-pipelines/generation-templates/use-dotnet-sdk.yml@self
   parameters:
     version: '9.x'
 
-- pwsh: dotnet tool install --global Microsoft.OpenApi.Hidi --version 1.6.24 
+- pwsh: dotnet tool install --global Microsoft.OpenApi.Hidi --version 1.6.24
   displayName: 'Install hidi tool'
 
 # verify that generated metadata is parsable as an Edm model

.azure-pipelines/generation-templates/capture-metadata.yml

@@ -1,9 +1,7 @@
-# Copyright (c) Microsoft Corporation.  All Rights Reserved.  Licensed under the MIT License.  See License in the project root for license information.
-# contains an end to end validation pipeline using C# compilation tests for staging beta metadata
-
-# capture-metadata.yml will result in the capture and cleaning of the metadata
-# with the artifacts posted to GitHub.
+# Copyright (c) Microsoft Corporation. All Rights Reserved. Licensed under the MIT License. See License in the project root for license information.
+# The following template creates multiple artifacts from a matrix, this is the intended behavior to follow the same workflow before the governance migration
 
+# For more information please refer to: https://github.com/microsoftgraph/MSGraph-SDK-Code-Generator/pull/1377#discussion_r2271024012
 parameters:
 - name: 'endpoint'
   type: string
@@ -13,7 +11,6 @@ parameters:
   default: $(System.ArtifactsDirectory)
 - name: 'cleanMetadataFolder'
   type: string
-
 jobs:
 - job: get_conversion_settings
   displayName: "Get conversion settings"
@@ -23,8 +20,7 @@ jobs:
     fetchDepth: 1
     persistCredentials: true
 
-  - template: checkout-metadata.yml
-
+  - template: /.azure-pipelines/generation-templates/checkout-metadata.yml@self
   - pwsh: |
       $dirPath = "./conversion-settings"
       $endpoint = "${{ parameters.endpoint }}"
@@ -41,38 +37,39 @@ jobs:
       Write-Host "##vso[task.setvariable variable=targets;isOutput=true]$json"
     name: setTargets
     workingDirectory: $(Build.SourcesDirectory)/msgraph-metadata
-
   - script: echo $(setTargets.targets)
     displayName: "Print settings"
-
 - job: convert_openapi
   dependsOn: get_conversion_settings
   displayName: Convert
   strategy:
     matrix: $[ dependencies.get_conversion_settings.outputs['setTargets.targets'] ]
   variables:
     targets: $[ dependencies.get_conversion_settings.outputs['setTargets.targets'] ]
-
+  templateContext:
+    outputs:
+    - output: pipelineArtifact
+      targetPath: '$(Build.ArtifactStagingDirectory)/$(Name)'
+      artifactName: '${{ parameters.cleanMetadataFolder }}-$(Name)'
   steps:
-  # We only need the scripts
   - checkout: self
     displayName: checkout generator
     fetchDepth: 1
     persistCredentials: true
 
-  - template: checkout-metadata.yml
+  - template: /.azure-pipelines/generation-templates/checkout-metadata.yml@self
 
   # required for the hidi to run
-  - template: use-dotnet-sdk.yml
+  - template: /.azure-pipelines/generation-templates/use-dotnet-sdk.yml@self
     parameters:
       version: '8.x'
 
-  # required for the hidi installation validation
-  - template: use-dotnet-sdk.yml
+# required for the hidi installation validation
+  - template: /.azure-pipelines/generation-templates/use-dotnet-sdk.yml@self
     parameters:
       version: '9.x'
 
-  - pwsh: dotnet tool install --global Microsoft.OpenApi.Hidi --version 1.6.24 
+  - pwsh: dotnet tool install --global Microsoft.OpenApi.Hidi --version 1.6.24
     displayName: install hidi
 
   - pwsh: |
@@ -90,14 +87,8 @@ jobs:
     inputs:
       sourceFolder: ${{ parameters.outputPath }}
       contents: '**/$(Name).yaml'
-      targetFolder: '$(Build.ArtifactStagingDirectory)'
+      targetFolder: '$(Build.ArtifactStagingDirectory)/$(Name)'
     displayName: Copy generated metadata
-
-  - task: PublishBuildArtifacts@1
-    inputs:
-      pathToPublish: '$(Build.ArtifactStagingDirectory)'
-      artifactName: ${{ parameters.cleanMetadataFolder }}
-
   - pwsh: |
       ./scripts/run-openapi-validation.ps1 -repoDirectory (Get-Location).Path -version "${{ parameters.endpoint }}" -platformName "$(Name)"
     displayName: ensure that OpenAPI docs can be parsed
@@ -107,21 +98,55 @@ jobs:
 - job: publish_openapi
   dependsOn: convert_openapi
   displayName: Publish
+  ## If there's new settings added please add them here too
+  templateContext:
+    inputs:
+    - input: pipelineArtifact
+      artifactName: '${{ parameters.cleanMetadataFolder }}-default'
+      targetPath: '$(Build.SourcesDirectory)/artifacts'
+    - input: pipelineArtifact
+      artifactName: '${{ parameters.cleanMetadataFolder }}-graphexplorer'
+      targetPath: '$(Build.SourcesDirectory)/artifacts'
+    - input: pipelineArtifact
+      artifactName: '${{ parameters.cleanMetadataFolder }}-openapi'
+      targetPath: '$(Build.SourcesDirectory)/artifacts'
+    - input: pipelineArtifact
+      artifactName: '${{ parameters.cleanMetadataFolder }}-powershell_v2'
+      targetPath: '$(Build.SourcesDirectory)/artifacts'
+    outputs:
+    - output: pipelineArtifact
+      targetPath: '$(Build.ArtifactStagingDirectory)'
+      artifactName: ${{ parameters.cleanMetadataFolder }}
   steps:
   # We only need the scripts
   - checkout: self
     displayName: checkout generator
     fetchDepth: 1
     persistCredentials: true
-
-  - template: checkout-metadata.yml
-  - template: set-user-config.yml
-  - task: DownloadPipelineArtifact@2
+    # Copy files from the maxtrix artifacts to a single folder
+  - task: CopyFiles@2
     inputs:
-      path: $(Build.SourcesDirectory)/msgraph-metadata/openapi/${{ parameters.endpoint }}
-      artifact: ${{ parameters.cleanMetadataFolder }}
+      sourceFolder: '$(Build.SourcesDirectory)/artifacts'
+      contents: '**/*.yaml'
+      targetFolder: '$(Build.ArtifactStagingDirectory)'
+      displayName: Copy artifact metadata
+    # Copy files from the target path where artifacts should be downloaded
+  - task: CopyFiles@2
+    inputs:
+      sourceFolder: '$(Build.ArtifactStagingDirectory)'
+      contents: '**/*.yaml'
+      targetFolder: '$(Build.SourcesDirectory)/msgraph-metadata/openapi/${{ parameters.endpoint }}'
+    displayName: Copy downloaded metadata
 
-  # Checkin clean metadata into metadata repo or make it an artifact.
+    # publish metadata as an artifact
+  - task: CopyFiles@2
+    inputs:
+      sourceFolder: ${{ parameters.outputPath }}
+      contents: '**/*.yaml'
+      targetFolder: '$(Build.ArtifactStagingDirectory)'
+    displayName: Copy downloaded metadata
+  - template: /.azure-pipelines/generation-templates/checkout-metadata.yml@self
+  - template: /.azure-pipelines/generation-templates/set-user-config.yml@self
   - pwsh: '$(scriptsDirectory)/git-push-cleanmetadata.ps1'
 
     displayName: push clean ${{ parameters.endpoint }} OpenAPI description to msgraph-metadata repo

.azure-pipelines/generation-templates/capture-openapi.yml

@@ -1,9 +1,5 @@
+# This Yaml Document has been converted by ESAI Yaml Pipeline Conversion Tool.
+# Please make sure to check all the converted content, it is your team's responsibility to make sure that the pipeline is still valid and functions as expected.
+# The 'DownloadBuildArtifacts@0' tasks have been converted to inputs within the `templateContext` section of each job.
 steps:
-- task: DownloadBuildArtifacts@0
-  inputs:
-    buildType: 'current'
-    downloadType: 'single'
-    artifactName: 'kiota'
-    downloadPath: '$(Build.SourcesDirectory)'
-
-- bash: chmod +x $(Build.SourcesDirectory)/kiota/kiota
+- bash: chmod +x $(Build.SourcesDirectory)/kiota/kiota

.azure-pipelines/generation-templates/download-kiota.yml

@@ -1,7 +1,4 @@
-steps:
-- task: DownloadBuildArtifacts@0
-  inputs:
-    buildType: 'current'
-    downloadType: 'single'
-    artifactName: 'typewriter'
-    downloadPath: '$(Build.SourcesDirectory)'
+# This Yaml Document has been converted by ESAI Yaml Pipeline Conversion Tool.
+# Please make sure to check all the converted content, it is your team's responsibility to make sure that the pipeline is still valid and functions as expected.
+# The 'DownloadBuildArtifacts@0' tasks have been converted to inputs within the `templateContext` section of each job.
+steps: []

.azure-pipelines/generation-templates/download-typewriter.yml

@@ -57,11 +57,9 @@ parameters:
   default: "-e '/copilot' -e '/copilot/**'"
 
 steps:
-- template: set-up-for-generation-kiota.yml
+- template: /.azure-pipelines/generation-templates/set-up-for-generation-kiota.yml@self
   parameters:
     cleanMetadataFolder: ${{ parameters.cleanMetadataFolder }}
-    downloadSteps:
-    - template: download-kiota.yml
 
 - checkout: ${{ parameters.repoName }}
   displayName: 'checkout ${{ parameters.repoName }}'
@@ -89,8 +87,16 @@ steps:
     RepoDirectory: $(Build.SourcesDirectory)/${{ parameters.repoName }}
     BranchName: ${{ parameters.branchName }}
   workingDirectory: ${{ parameters.repoName }}
-
-- bash: '$(kiotaDirectory)/kiota generate --openapi $(Build.SourcesDirectory)/msgraph-metadata/${{ parameters.cleanMetadataFolder }}/openapi.yaml --language ${{ parameters.language }} -o $(kiotaDirectory)/output -n ${{ parameters.targetNamespace }} -c ${{ parameters.targetClassName }} ${{ parameters.customArguments }} ${{ parameters.pathExclusionArguments }}'
+- pwsh: |
+    Write-Host "=== Files in Kiota Directory ==="
+    Write-Host "Kiota Directory: $(kiotaDirectory)"
+    if (Test-Path "$(kiotaDirectory)") {
+        Get-ChildItem "$(kiotaDirectory)" -Recurse | Format-Table Name, FullName, Length, LastWriteTime -AutoSize
+    } else {
+        Write-Warning "Kiota directory not found: $(kiotaDirectory)"
+    }
+  displayName: 'List files in Kiota directory'
+- pwsh: '& "$(kiotaDirectory)/kiota.exe" generate --openapi "$(Build.SourcesDirectory)/msgraph-metadata/${{ parameters.cleanMetadataFolder }}/openapi.yaml" --language "${{ parameters.language }}" -o "$(kiotaDirectory)/output" -n "${{ parameters.targetNamespace }}" -c "${{ parameters.targetClassName }}" ${{ parameters.customArguments }} ${{ parameters.pathExclusionArguments }}'
   displayName: 'Run Kiota for ${{ parameters.language }} ${{ parameters.version }}'
   env:
     KIOTA_GENERATION:EXPORTPUBLICAPI: ${{ parameters.exportDom }}

.azure-pipelines/generation-templates/language-generation-kiota.yml

@@ -34,12 +34,9 @@ parameters:
   displayName: Generate PR after publish changes?
 
 steps:
-- template: set-up-for-generation.yml
+- template: /.azure-pipelines/generation-templates/set-up-for-generation.yml@self
   parameters:
     cleanMetadataFolder: ${{ parameters.cleanMetadataFolder }}
-    downloadSteps:
-    - template: download-typewriter.yml
-
 - checkout: ${{ parameters.repoName }}
   displayName: 'checkout ${{ parameters.repoName }}'
   fetchDepth: 1
@@ -67,7 +64,7 @@ steps:
     TypewriterExecutable: $(typewriterExecutable)
     TypewriterDirectory: $(typewriterDirectory)
     Language: ${{ parameters.language }}
-    Endpoint:  ${{ parameters.version }}
+    Endpoint: ${{ parameters.version }}
   displayName: 'Run Typewriter for ${{ parameters.language }} ${{ parameters.version }}'
 
 - ${{ parameters.languageSpecificSteps }}

.azure-pipelines/generation-templates/language-generation.yml

@@ -1,20 +1,13 @@
 parameters:
-  - name: cleanMetadataFolder
-    type: string
-  - name: downloadSteps
-    type: stepList
-    default:
-    - pwsh: |
-        Write-Error "No download step is provided"
+- name: cleanMetadataFolder
+  type: string
 
 steps:
-- template: set-user-config.yml
-- template: use-dotnet-sdk.yml
+- template: /.azure-pipelines/generation-templates/set-user-config.yml@self
+- template: /.azure-pipelines/generation-templates/use-dotnet-sdk.yml@self
   parameters:
     version: "9.x" #kiota uses a net9 target
 
-- ${{ parameters.downloadSteps }}
-
 # checkout metadata repo if capture and clean step is skipped
 - checkout: msgraph-metadata
   displayName: checkout metadata
@@ -30,16 +23,3 @@ steps:
     Move-Item -Path $(Build.SourcesDirectory)/msgraph-metadata/openapi/beta/openapi.yaml -Destination $(Build.SourcesDirectory)/msgraph-metadata/clean_beta_openapi/
   displayName: Move metadata to expected location
   condition: or(eq(variables.skipMetadataCaptureAndClean, true), eq(variables.skipOpenApiCaptureAndClean, true))
-
-# if capture and clean step is not skipped
-# then download the artifact from capture and clean steps
-# follow the same folder structure as msgraph-metadata repo
-# so that metadata reference path is always the same
-- task: DownloadBuildArtifacts@0
-  inputs:
-    buildType: 'current'
-    downloadType: 'single'
-    artifactName: ${{ parameters.cleanMetadataFolder }}
-    downloadPath: '$(Build.SourcesDirectory)/msgraph-metadata'
-  condition: and(eq(variables.skipMetadataCaptureAndClean, false), eq(variables.skipOpenApiCaptureAndClean, false))
-  displayName: Downloading metadata from artifacts