ci: fix broken OpenAPI generation

Author: MIchaelMainer

.azure-pipelines/generation-templates/capture-openapi.yml

@@ -1,3 +1,4 @@
+# capture-openapi.yml
 # Copyright (c) Microsoft Corporation. All Rights Reserved. Licensed under the MIT License. See License in the project root for license information.
 # The following template creates multiple artifacts from a matrix, this is the intended behavior to follow the same workflow before the governance migration
 
@@ -21,6 +22,7 @@ jobs:
     persistCredentials: true
 
   - template: /.azure-pipelines/generation-templates/checkout-metadata.yml@self
+
   - pwsh: |
       $dirPath = "./conversion-settings"
       $endpoint = "${{ parameters.endpoint }}"
@@ -37,8 +39,10 @@ jobs:
       Write-Host "##vso[task.setvariable variable=targets;isOutput=true]$json"
     name: setTargets
     workingDirectory: $(Build.SourcesDirectory)/msgraph-metadata
+
   - script: echo $(setTargets.targets)
     displayName: "Print settings"
+
 - job: convert_openapi
   dependsOn: get_conversion_settings
   displayName: Convert
@@ -58,28 +62,33 @@ jobs:
     persistCredentials: true
 
   - template: /.azure-pipelines/generation-templates/checkout-metadata.yml@self
-
   # required for the hidi to run
   - template: /.azure-pipelines/generation-templates/use-dotnet-sdk.yml@self
     parameters:
       version: '8.x'
-
-# required for the hidi installation validation
+    displayName: use dotnet sdk 8.x
+  # required for the hidi installation validation
   - template: /.azure-pipelines/generation-templates/use-dotnet-sdk.yml@self
     parameters:
       version: '9.x'
+    displayName: use dotnet sdk 9.x
 
   - pwsh: dotnet tool install --global Microsoft.OpenApi.Hidi --version 1.6.24
     displayName: install hidi
 
   - pwsh: |
-      git fetch origin master
-      git switch master
+      git fetch origin master | Write-Host -ForegroundColor Yellow
+      git switch master | Write-Host -ForegroundColor Yellow
     displayName: ensure the generation happens from master latest
     workingDirectory: $(Build.SourcesDirectory)/msgraph-metadata
 
-  - pwsh: '$(scriptsDirectory)/generate-open-api.ps1 -endpointVersion ${{ parameters.endpoint }} -settings "$(conversionSettingsDirectory)/$(File)" -platformName "$(Name)"'
-    displayName: 'update ${{ parameters.endpoint }} open API description'
+  - pwsh: |
+      Write-Host "`nGet status before generation:" -ForegroundColor Green
+      git status | Write-Host -ForegroundColor Yellow
+      $(scriptsDirectory)/generate-open-api.ps1 -endpointVersion ${{ parameters.endpoint }} -settings "$(conversionSettingsDirectory)/$(File)" -platformName "$(Name)"
+      Write-Host "`nGet status after generation:" -ForegroundColor Green
+      git status | Write-Host -ForegroundColor Yellow
+    displayName: 'generate ${{ parameters.endpoint }} open API description'
     workingDirectory: $(Build.SourcesDirectory)/msgraph-metadata
 
   # publish metadata as an artifact
@@ -88,16 +97,16 @@ jobs:
       sourceFolder: ${{ parameters.outputPath }}
       contents: '**/$(Name).yaml'
       targetFolder: '$(Build.ArtifactStagingDirectory)/$(Name)'
-    displayName: Copy generated metadata
+    displayName: Copy generated OpenAPI yaml file to ArtifactStagingDirectory
+
   - pwsh: |
       ./scripts/run-openapi-validation.ps1 -repoDirectory (Get-Location).Path -version "${{ parameters.endpoint }}" -platformName "$(Name)"
-    displayName: ensure that OpenAPI docs can be parsed
+    displayName: Validate that OpenAPI docs can be parsed
     workingDirectory: $(Build.SourcesDirectory)/msgraph-metadata
 
-
 - job: publish_openapi
   dependsOn: convert_openapi
-  displayName: Publish
+  displayName: Publish OpenAPI files
   ## If there's new settings added please add them here too
   templateContext:
     inputs:
@@ -123,32 +132,31 @@ jobs:
     displayName: checkout generator
     fetchDepth: 1
     persistCredentials: true
-    # Copy files from the maxtrix artifacts to a single folder
+
+  #  Note that msgraph-metadata repo has detached HEAD
+  - template: /.azure-pipelines/generation-templates/checkout-metadata.yml@self    
+
+  - template: /.azure-pipelines/generation-templates/set-user-config.yml@self  
+
+  # Copy OpenAPI files from the matrix artifacts to the ArtifactStagingDirectory
+  # This is used for library generation later
   - task: CopyFiles@2
     inputs:
       sourceFolder: '$(Build.SourcesDirectory)/artifacts'
       contents: '**/*.yaml'
       targetFolder: '$(Build.ArtifactStagingDirectory)'
-      displayName: Copy artifact metadata
-    # Copy files from the target path where artifacts should be downloaded
+    displayName: Copy OpenAPI input artifacts to artifact staging directory
+    
+  # Copy OpenAPI files from ArtifactStagingDirectory to local msgraph-metadata repo
   - task: CopyFiles@2
     inputs:
       sourceFolder: '$(Build.ArtifactStagingDirectory)'
       contents: '**/*.yaml'
       targetFolder: '$(Build.SourcesDirectory)/msgraph-metadata/openapi/${{ parameters.endpoint }}'
-    displayName: Copy downloaded metadata
+    displayName: Copy OpenAPI files to local msgraph-metadata repo
 
-    # publish metadata as an artifact
-  - task: CopyFiles@2
-    inputs:
-      sourceFolder: ${{ parameters.outputPath }}
-      contents: '**/*.yaml'
-      targetFolder: '$(Build.ArtifactStagingDirectory)'
-    displayName: Copy downloaded metadata
-  - template: /.azure-pipelines/generation-templates/checkout-metadata.yml@self
-  - template: /.azure-pipelines/generation-templates/set-user-config.yml@self
+  # Push changes to msgraph-metadata repo
   - pwsh: '$(scriptsDirectory)/git-push-cleanmetadata.ps1'
-
     displayName: push clean ${{ parameters.endpoint }} OpenAPI description to msgraph-metadata repo
     env:
       EndpointVersion: ${{ parameters.endpoint }}

scripts/git-push-cleanmetadata.ps1

@@ -1,31 +1,61 @@
+# git-push-cleanmetadata.ps1
+# Copyright (c) Microsoft Corporation.  All Rights Reserved.  Licensed under the MIT License.  See License in the 
+# project root for license information.
+
+# This script stashes any changes, checks out the latest master branch, applies the stashed changes, commits, and 
+# pushes the changes back to the remote repository.
+
 if ($env:PublishChanges -eq $False)
 {
-    Write-Host "Not publishing changes per the run parameter!" -ForegroundColor Green
+    Write-Host "Not publishing changes per the run parameter!" -ForegroundColor Red
     return;
 }
 
-Write-Host "About to add clean $env:EndpointVersion metadata file....."
+Write-Host "`nGet status:" -ForegroundColor Green
+git status | Write-Host -ForegroundColor Yellow
+
+Write-Host "`nStash the update metadata files.....`nRunning: git stash" -ForegroundColor Green
+git stash | Write-Host -ForegroundColor Yellow
 
-git fetch origin master
+Write-Host "`nFetching latest master branch to ensure we are up to date..." -ForegroundColor Green
+git fetch origin master | Write-Host -ForegroundColor Yellow
 # checkout master to move from detached HEAD mode
-git switch master
+git switch master | Write-Host -ForegroundColor Yellow
+
+Write-Host "`nGet status:" -ForegroundColor Green
+git status | Write-Host -ForegroundColor Yellow
+
+Write-Host "`nApply stashed metadata files...`nRunning: git stash pop" -ForegroundColor Green
+git stash pop | Write-Host -ForegroundColor Yellow
+
+Write-Host "`nGet status:" -ForegroundColor Green
+git status | Write-Host -ForegroundColor Yellow
+
+Write-Host "`nStaging clean $env:EndpointVersion metadata files....." -ForegroundColor Green
+git add . | Write-Host -ForegroundColor Yellow
+
+Write-Host "`nGet status:" -ForegroundColor Green
+git status | Write-Host -ForegroundColor Yellow
+
+Write-Host "`nAttempting to commit clean $env:EndpointVersion metadata files....." -ForegroundColor Green
 
-git add . | Write-Host
 if ($env:BUILD_REASON -eq 'Manual') # Skip CI if manually running this pipeline.
 {
-    git commit -m "Update clean metadata file with $env:BUILD_BUILDID [skip ci]" | Write-Host
+    git commit -m "Update clean metadata file with $env:BUILD_BUILDID [skip ci]" | Write-Host -ForegroundColor Yellow
 }
 else
 {
-    git commit -m "Update clean metadata file with $env:BUILD_BUILDID" | Write-Host
+    git commit -m "Update clean metadata file with $env:BUILD_BUILDID" | Write-Host -ForegroundColor Yellow
 }
 
-Write-Host "Added and commited cleaned $env:EndpointVersion metadata." -ForegroundColor Green
+Write-Host "`nGet status:" -ForegroundColor Green
+git status | Write-Host -ForegroundColor Yellow
 
+Write-Host "`nRunning: git pull origin master --rebase..." -ForegroundColor Green
 # sync branch before pushing
 # this is especially important while running v1 and beta in parallel
 # and one process goes out of sync because of the other's check-in
-git pull origin master --rebase
+git pull origin master --rebase | Write-Host -ForegroundColor Yellow
 
-git push --set-upstream origin master | Write-Host
-Write-Host "Pushed the results of the build $env:BUILD_BUILDID to the master branch." -ForegroundColor Green
+Write-Host "`nRunning: git push --set-upstream origin master ..." -ForegroundColor Green
+git push --set-upstream origin master | Write-Host -ForegroundColor Yellow