ci: fix broken OpenAPI generation (#1392)

* ci: fix broken OpenAPI generation
* ci: add PR for OpenAPI files
* docs: clarify comments
* Update scripts/git-push-cleanmetadata.ps1
* ci: clarify use of version when creating PR
* ci: remove display name from template use
* ci update logging
* ci: correct the copyfile overwrite parameter
* ci: add approval step for OpenAPI before running Kiota generation
* ci: update env var name
* ci: typings gen does not use open_api
* ci: name branch using endpointversion, add clarity for host output
* ci: update for supporting builds
* ci: add PR title for OpenAPI PRs
* ci: add beta openapi approval step

Author: MIchaelMainer

.azure-pipelines/generation-pipeline.yml

@@ -298,10 +298,45 @@ extends:
           outputPath: $(cleanOpenAPIFileBetaOutputPath)
           cleanMetadataFolder: $(cleanOpenAPIFolderBeta)
 
+    # Approval stage for v1 OpenAPI generation. All subsequent stages depending on OpenAPI generation
+    # will be blocked until approval is granted. Any new stages depending on OpenAPI generation
+    # should also depend on this approval stage.
+    - stage: open_api_v1_approval
+      dependsOn:
+      - stage_v1_openapi
+      condition: eq(dependencies.stage_v1_openapi.result, 'Succeeded')
+      jobs:
+      - deployment: OpenApiApproval
+        displayName: 'OpenAPI v1.0 Generation Review'
+        environment: 'openapi-generation-review'
+        strategy:
+          runOnce:
+            deploy:
+              steps:
+              - script: echo "OpenAPI generation approved."
+
+    # Approval stage for beta OpenAPI generation. All subsequent stages depending on OpenAPI generation
+    # will be blocked until approval is granted. Any new stages depending on OpenAPI generation
+    # should also depend on this approval stage.
+    - stage: open_api_beta_approval
+      dependsOn:
+      - stage_beta_openapi
+      condition: eq(dependencies.stage_beta_openapi.result, 'Succeeded')
+      jobs:
+      - deployment: OpenApiApproval
+        displayName: 'OpenAPI Beta Generation Review'
+        environment: 'openapi-generation-review'
+        strategy:
+          runOnce:
+            deploy:
+              steps:
+              - script: echo "OpenAPI generation approved."              
+
     - stage: stage_csharp_v1_kiota
       dependsOn:
       - stage_build_and_publish_kiota
       - stage_v1_openapi
+      - open_api_v1_approval
       condition: |
         and
         (
@@ -343,6 +378,7 @@ extends:
       dependsOn:
       - stage_build_and_publish_kiota
       - stage_beta_openapi
+      - open_api_beta_approval
       condition: |
         and
         (
@@ -384,6 +420,7 @@ extends:
       dependsOn:
       - stage_build_and_publish_kiota
       - stage_v1_openapi
+      - open_api_v1_approval
       condition: |
         and
         (
@@ -423,6 +460,7 @@ extends:
       dependsOn:
       - stage_build_and_publish_kiota
       - stage_beta_openapi
+      - open_api_beta_approval
       condition: |
         and
         (
@@ -462,6 +500,7 @@ extends:
       dependsOn:
       - stage_build_and_publish_kiota
       - stage_v1_openapi
+      - open_api_v1_approval
       condition: |
         and
         (
@@ -503,6 +542,7 @@ extends:
       dependsOn:
       - stage_build_and_publish_kiota
       - stage_beta_openapi
+      - open_api_beta_approval
       condition: |
         and
         (
@@ -544,6 +584,7 @@ extends:
       dependsOn:
       - stage_build_and_publish_kiota
       - stage_beta_openapi
+      - open_api_beta_approval
       condition: |
         and
         (
@@ -585,6 +626,7 @@ extends:
       dependsOn:
       - stage_build_and_publish_kiota
       - stage_v1_openapi
+      - open_api_v1_approval
       condition: |
         and
         (
@@ -700,6 +742,7 @@ extends:
       dependsOn:
       - stage_build_and_publish_kiota
       - stage_v1_openapi
+      - open_api_v1_approval
       condition: |
         and
         (
@@ -742,6 +785,7 @@ extends:
       dependsOn:
       - stage_build_and_publish_kiota
       - stage_beta_openapi
+      - open_api_beta_approval
       condition: |
         and
         (
@@ -784,6 +828,7 @@ extends:
       dependsOn:
       - stage_build_and_publish_kiota
       - stage_v1_openapi
+      - open_api_v1_approval
       condition: |
         and
         (
@@ -824,6 +869,7 @@ extends:
       dependsOn:
       - stage_build_and_publish_kiota
       - stage_beta_openapi
+      - open_api_beta_approval
       condition: |
         and
         (

.azure-pipelines/generation-templates/capture-metadata.yml

@@ -166,9 +166,9 @@ steps:
 
 # Checkin clean metadata into metadata repo or make it an artifact.
 - pwsh: '$(scriptsDirectory)/git-push-cleanmetadata.ps1'
-
-  displayName: push clean ${{ parameters.endpoint }} metadata to msgraph-metadata repo
+  displayName: push clean ${{ parameters.endpoint }} CSDL metadata to msgraph-metadata repo
   env:
+    CreateOpenAPIPR: False
     EndpointVersion: ${{ parameters.endpoint }}
     PublishChanges: $(publishChanges)
   workingDirectory: '$(Build.SourcesDirectory)/msgraph-metadata'

.azure-pipelines/generation-templates/capture-openapi.yml

@@ -1,3 +1,4 @@
+# capture-openapi.yml
 # Copyright (c) Microsoft Corporation. All Rights Reserved. Licensed under the MIT License. See License in the project root for license information.
 # The following template creates multiple artifacts from a matrix, this is the intended behavior to follow the same workflow before the governance migration
 
@@ -21,6 +22,7 @@ jobs:
     persistCredentials: true
 
   - template: /.azure-pipelines/generation-templates/checkout-metadata.yml@self
+
   - pwsh: |
       $dirPath = "./conversion-settings"
       $endpoint = "${{ parameters.endpoint }}"
@@ -37,8 +39,10 @@ jobs:
       Write-Host "##vso[task.setvariable variable=targets;isOutput=true]$json"
     name: setTargets
     workingDirectory: $(Build.SourcesDirectory)/msgraph-metadata
+
   - script: echo $(setTargets.targets)
     displayName: "Print settings"
+
 - job: convert_openapi
   dependsOn: get_conversion_settings
   displayName: Convert
@@ -58,13 +62,12 @@ jobs:
     persistCredentials: true
 
   - template: /.azure-pipelines/generation-templates/checkout-metadata.yml@self
-
   # required for the hidi to run
   - template: /.azure-pipelines/generation-templates/use-dotnet-sdk.yml@self
     parameters:
       version: '8.x'
 
-# required for the hidi installation validation
+  # required for the hidi installation validation
   - template: /.azure-pipelines/generation-templates/use-dotnet-sdk.yml@self
     parameters:
       version: '9.x'
@@ -78,8 +81,13 @@ jobs:
     displayName: ensure the generation happens from master latest
     workingDirectory: $(Build.SourcesDirectory)/msgraph-metadata
 
-  - pwsh: '$(scriptsDirectory)/generate-open-api.ps1 -endpointVersion ${{ parameters.endpoint }} -settings "$(conversionSettingsDirectory)/$(File)" -platformName "$(Name)"'
-    displayName: 'update ${{ parameters.endpoint }} open API description'
+  - pwsh: |
+      Write-Host "`ngit status before generation:"
+      git status
+      $(scriptsDirectory)/generate-open-api.ps1 -endpointVersion ${{ parameters.endpoint }} -settings "$(conversionSettingsDirectory)/$(File)" -platformName "$(Name)"
+      Write-Host "`ngit status after generation:"
+      git status
+    displayName: 'generate ${{ parameters.endpoint }} open API description'
     workingDirectory: $(Build.SourcesDirectory)/msgraph-metadata
 
   # publish metadata as an artifact
@@ -88,16 +96,16 @@ jobs:
       sourceFolder: ${{ parameters.outputPath }}
       contents: '**/$(Name).yaml'
       targetFolder: '$(Build.ArtifactStagingDirectory)/$(Name)'
-    displayName: Copy generated metadata
+    displayName: Copy generated OpenAPI yaml file to ArtifactStagingDirectory
+
   - pwsh: |
       ./scripts/run-openapi-validation.ps1 -repoDirectory (Get-Location).Path -version "${{ parameters.endpoint }}" -platformName "$(Name)"
-    displayName: ensure that OpenAPI docs can be parsed
+    displayName: Validate that OpenAPI docs can be parsed
     workingDirectory: $(Build.SourcesDirectory)/msgraph-metadata
 
-
 - job: publish_openapi
   dependsOn: convert_openapi
-  displayName: Publish
+  displayName: Publish OpenAPI files
   ## If there's new settings added please add them here too
   templateContext:
     inputs:
@@ -123,35 +131,61 @@ jobs:
     displayName: checkout generator
     fetchDepth: 1
     persistCredentials: true
-    # Copy files from the maxtrix artifacts to a single folder
+
+  #  Note that msgraph-metadata repo has detached HEAD
+  - template: /.azure-pipelines/generation-templates/checkout-metadata.yml@self    
+
+  - template: /.azure-pipelines/generation-templates/set-user-config.yml@self  
+
+  # Copy OpenAPI files from the matrix artifacts to the ArtifactStagingDirectory
+  # This is used for library generation later
   - task: CopyFiles@2
     inputs:
       sourceFolder: '$(Build.SourcesDirectory)/artifacts'
       contents: '**/*.yaml'
       targetFolder: '$(Build.ArtifactStagingDirectory)'
-      displayName: Copy artifact metadata
-    # Copy files from the target path where artifacts should be downloaded
+    displayName: Copy OpenAPI input artifacts to artifact staging directory
+    
+  # Copy OpenAPI files from ArtifactStagingDirectory to local msgraph-metadata repo
   - task: CopyFiles@2
     inputs:
       sourceFolder: '$(Build.ArtifactStagingDirectory)'
       contents: '**/*.yaml'
       targetFolder: '$(Build.SourcesDirectory)/msgraph-metadata/openapi/${{ parameters.endpoint }}'
-    displayName: Copy downloaded metadata
+      overwrite: true
+    displayName: Copy OpenAPI files to local msgraph-metadata repo
 
-    # publish metadata as an artifact
-  - task: CopyFiles@2
-    inputs:
-      sourceFolder: ${{ parameters.outputPath }}
-      contents: '**/*.yaml'
-      targetFolder: '$(Build.ArtifactStagingDirectory)'
-    displayName: Copy downloaded metadata
-  - template: /.azure-pipelines/generation-templates/checkout-metadata.yml@self
-  - template: /.azure-pipelines/generation-templates/set-user-config.yml@self
+  # Push changes to msgraph-metadata repo
   - pwsh: '$(scriptsDirectory)/git-push-cleanmetadata.ps1'
-
-    displayName: push clean ${{ parameters.endpoint }} OpenAPI description to msgraph-metadata repo
+    displayName: Publish ${{ parameters.endpoint }} OpenAPI description to msgraph-metadata repo
     env:
+      CreateOpenAPIPR: True
       EndpointVersion: ${{ parameters.endpoint }}
       PublishChanges: $(publishChanges)
     workingDirectory: '$(Build.SourcesDirectory)/msgraph-metadata'
     enabled: true
+    
+
+  # Create PR - note that this PR is not used for gating. It's just for discovery purposes. 
+  # Library generation PRs will still be created based on the OpenAPI files.
+
+  - task: AzureKeyVault@2
+    displayName: "Azure Key Vault: Get Secrets"
+    inputs:
+      azureSubscription: "Federated AKV Managed Identity Connection"
+      KeyVaultName: akv-prod-eastus
+      SecretsFilter: "microsoft-graph-devx-bot-appid,microsoft-graph-devx-bot-privatekey"
+
+  - pwsh: '$(scriptsDirectory)/create-pull-request.ps1'
+    displayName: 'Create Pull Request for the generated OpenAPI files for msgraph-metadata'
+    env:
+      BaseBranch: master
+      GeneratePullRequest: true
+      GhAppId: $(microsoft-graph-devx-bot-appid)
+      GhAppKey: $(microsoft-graph-devx-bot-privatekey)
+      OverrideSkipCI: false
+      RepoName: 'microsoftgraph/msgraph-metadata'
+      ScriptsDirectory: $(scriptsDirectory)
+      # Version is intentionally left empty for OpenAPI PRs as versioning is not applicable in this context.
+      Version: ''
+    workingDirectory: '$(Build.SourcesDirectory)/msgraph-metadata'

.azure-pipelines/generation-templates/use-dotnet-sdk.yml

@@ -5,7 +5,7 @@ parameters:
 
 steps:
 - task: UseDotNet@2
-  displayName: 'Use .NET SDK'
+  displayName: 'Use .NET SDK ${{ parameters.version }}'
   inputs:
     packageType: sdk
     version: ${{ parameters.version }}

scripts/create-pull-request.ps1

@@ -1,24 +1,27 @@
 if (($env:OverrideSkipCI -eq $False) -and ($env:BUILD_REASON -eq 'Manual')) # Skip CI if manually running this pipeline.
 {
-    Write-Host "Skipping pull request creation due Skip CI." -ForegroundColor Green
+    Write-Host "Skipping pull request creation due Skip CI."
     return;
 }
 
 if (($env:GeneratePullRequest -eq $False)) { # Skip CI if manually running this pipeline.
-    Write-Host "Skipping pull request creation due this repository being disabled" -ForegroundColor Green
+    Write-Host "Skipping pull request creation due this repository being disabled"
     return;
 }
 
-# Special case for beta typings as it uses a non-conforming preview versioning.
+# Special case for beta typings as it uses a non-conforming preview versioning. Helps with triggering Release Please.
 if ($env:RepoName.Contains("msgraph-beta-typescript-typings"))
 {
-    $title = "feat: generated $version models and request builders"    
+    $title = "feat: generated $env:Version models and request builders"    
+}
+elseif ($env:RepoName.Contains("msgraph-metadata")) # we are only generating OpenAPI PRs for the metadata repo
+{
+    $title = "Generated $env:Version OpenAPI descriptions"
 }
 else {
-    $title = "Generated $version models and request builders"
+    $title = "Generated $env:Version models and request builders"
 }
 
-$version = $env:Version
 $body = ":bangbang:**_Important_**:bangbang: <br> Check for unexpected deletions or changes in this PR and ensure relevant CI checks are passing. <br><br> **Note:** This pull request was automatically created by Azure pipelines."
 $baseBranchParameter = ""
 
@@ -30,10 +33,10 @@ if (![string]::IsNullOrEmpty($env:BaseBranch))
 # The installed application is required to have the following permissions: read/write on pull requests/
 $tokenGenerationScript = "$env:ScriptsDirectory\Generate-Github-Token.ps1"
 $env:GITHUB_TOKEN = & $tokenGenerationScript -AppClientId $env:GhAppId -AppPrivateKeyContents $env:GhAppKey -Repository $env:RepoName
-Write-Host "Fetched Github Token for PR generation and set as environment variable." -ForegroundColor Green
+Write-Host "Fetched Github Token for PR generation and set as environment variable."
 
 # No need to specify reviewers as code owners should be added automatically.
 Invoke-Expression "gh auth login" # login to GitHub
 Invoke-Expression "gh pr create -t ""$title"" -b ""$body"" $baseBranchParameter | Write-Host"
 
-Write-Host "Pull Request Created successfully." -ForegroundColor Green
+Write-Host "Pull Request Created successfully."