diff --git a/.azure-pipelines/generation-pipeline.yml b/.azure-pipelines/generation-pipeline.yml
index c377407b3..46e179ea7 100644
--- a/.azure-pipelines/generation-pipeline.yml
+++ b/.azure-pipelines/generation-pipeline.yml
@@ -298,10 +298,45 @@ extends:
outputPath: $(cleanOpenAPIFileBetaOutputPath)
cleanMetadataFolder: $(cleanOpenAPIFolderBeta)
+ # Approval stage for v1 OpenAPI generation. All subsequent stages depending on OpenAPI generation
+ # will be blocked until approval is granted. Any new stages depending on OpenAPI generation
+ # should also depend on this approval stage.
+ - stage: open_api_v1_approval
+ dependsOn:
+ - stage_v1_openapi
+ condition: eq(dependencies.stage_v1_openapi.result, 'Succeeded')
+ jobs:
+ - deployment: OpenApiApproval
+ displayName: 'OpenAPI v1.0 Generation Review'
+ environment: 'openapi-generation-review'
+ strategy:
+ runOnce:
+ deploy:
+ steps:
+ - script: echo "OpenAPI generation approved."
+
+ # Approval stage for beta OpenAPI generation. All subsequent stages depending on OpenAPI generation
+ # will be blocked until approval is granted. Any new stages depending on OpenAPI generation
+ # should also depend on this approval stage.
+ - stage: open_api_beta_approval
+ dependsOn:
+ - stage_beta_openapi
+ condition: eq(dependencies.stage_beta_openapi.result, 'Succeeded')
+ jobs:
+ - deployment: OpenApiApproval
+ displayName: 'OpenAPI Beta Generation Review'
+ environment: 'openapi-generation-review'
+ strategy:
+ runOnce:
+ deploy:
+ steps:
+ - script: echo "OpenAPI generation approved."
+
- stage: stage_csharp_v1_kiota
dependsOn:
- stage_build_and_publish_kiota
- stage_v1_openapi
+ - open_api_v1_approval
condition: |
and
(
@@ -343,6 +378,7 @@ extends:
dependsOn:
- stage_build_and_publish_kiota
- stage_beta_openapi
+ - open_api_beta_approval
condition: |
and
(
@@ -384,6 +420,7 @@ extends:
dependsOn:
- stage_build_and_publish_kiota
- stage_v1_openapi
+ - open_api_v1_approval
condition: |
and
(
@@ -423,6 +460,7 @@ extends:
dependsOn:
- stage_build_and_publish_kiota
- stage_beta_openapi
+ - open_api_beta_approval
condition: |
and
(
@@ -462,6 +500,7 @@ extends:
dependsOn:
- stage_build_and_publish_kiota
- stage_v1_openapi
+ - open_api_v1_approval
condition: |
and
(
@@ -503,6 +542,7 @@ extends:
dependsOn:
- stage_build_and_publish_kiota
- stage_beta_openapi
+ - open_api_beta_approval
condition: |
and
(
@@ -544,6 +584,7 @@ extends:
dependsOn:
- stage_build_and_publish_kiota
- stage_beta_openapi
+ - open_api_beta_approval
condition: |
and
(
@@ -585,6 +626,7 @@ extends:
dependsOn:
- stage_build_and_publish_kiota
- stage_v1_openapi
+ - open_api_v1_approval
condition: |
and
(
@@ -700,6 +742,7 @@ extends:
dependsOn:
- stage_build_and_publish_kiota
- stage_v1_openapi
+ - open_api_v1_approval
condition: |
and
(
@@ -742,6 +785,7 @@ extends:
dependsOn:
- stage_build_and_publish_kiota
- stage_beta_openapi
+ - open_api_beta_approval
condition: |
and
(
@@ -784,6 +828,7 @@ extends:
dependsOn:
- stage_build_and_publish_kiota
- stage_v1_openapi
+ - open_api_v1_approval
condition: |
and
(
@@ -824,6 +869,7 @@ extends:
dependsOn:
- stage_build_and_publish_kiota
- stage_beta_openapi
+ - open_api_beta_approval
condition: |
and
(
diff --git a/.azure-pipelines/generation-templates/capture-metadata.yml b/.azure-pipelines/generation-templates/capture-metadata.yml
index 96637da28..4a6fb2377 100644
--- a/.azure-pipelines/generation-templates/capture-metadata.yml
+++ b/.azure-pipelines/generation-templates/capture-metadata.yml
@@ -166,9 +166,9 @@ steps:
# Checkin clean metadata into metadata repo or make it an artifact.
- pwsh: '$(scriptsDirectory)/git-push-cleanmetadata.ps1'
-
- displayName: push clean ${{ parameters.endpoint }} metadata to msgraph-metadata repo
+ displayName: push clean ${{ parameters.endpoint }} CSDL metadata to msgraph-metadata repo
env:
+ CreateOpenAPIPR: False
EndpointVersion: ${{ parameters.endpoint }}
PublishChanges: $(publishChanges)
workingDirectory: '$(Build.SourcesDirectory)/msgraph-metadata'
diff --git a/.azure-pipelines/generation-templates/capture-openapi.yml b/.azure-pipelines/generation-templates/capture-openapi.yml
index 617931c8d..499723f06 100644
--- a/.azure-pipelines/generation-templates/capture-openapi.yml
+++ b/.azure-pipelines/generation-templates/capture-openapi.yml
@@ -1,3 +1,4 @@
+# capture-openapi.yml
# Copyright (c) Microsoft Corporation. All Rights Reserved. Licensed under the MIT License. See License in the project root for license information.
# The following template creates multiple artifacts from a matrix, this is the intended behavior to follow the same workflow before the governance migration
@@ -21,6 +22,7 @@ jobs:
persistCredentials: true
- template: /.azure-pipelines/generation-templates/checkout-metadata.yml@self
+
- pwsh: |
$dirPath = "./conversion-settings"
$endpoint = "${{ parameters.endpoint }}"
@@ -37,8 +39,10 @@ jobs:
Write-Host "##vso[task.setvariable variable=targets;isOutput=true]$json"
name: setTargets
workingDirectory: $(Build.SourcesDirectory)/msgraph-metadata
+
- script: echo $(setTargets.targets)
displayName: "Print settings"
+
- job: convert_openapi
dependsOn: get_conversion_settings
displayName: Convert
@@ -58,13 +62,12 @@ jobs:
persistCredentials: true
- template: /.azure-pipelines/generation-templates/checkout-metadata.yml@self
-
# required for the hidi to run
- template: /.azure-pipelines/generation-templates/use-dotnet-sdk.yml@self
parameters:
version: '8.x'
-# required for the hidi installation validation
+ # required for the hidi installation validation
- template: /.azure-pipelines/generation-templates/use-dotnet-sdk.yml@self
parameters:
version: '9.x'
@@ -78,8 +81,13 @@ jobs:
displayName: ensure the generation happens from master latest
workingDirectory: $(Build.SourcesDirectory)/msgraph-metadata
- - pwsh: '$(scriptsDirectory)/generate-open-api.ps1 -endpointVersion ${{ parameters.endpoint }} -settings "$(conversionSettingsDirectory)/$(File)" -platformName "$(Name)"'
- displayName: 'update ${{ parameters.endpoint }} open API description'
+ - pwsh: |
+ Write-Host "`ngit status before generation:"
+ git status
+ $(scriptsDirectory)/generate-open-api.ps1 -endpointVersion ${{ parameters.endpoint }} -settings "$(conversionSettingsDirectory)/$(File)" -platformName "$(Name)"
+ Write-Host "`ngit status after generation:"
+ git status
+ displayName: 'generate ${{ parameters.endpoint }} open API description'
workingDirectory: $(Build.SourcesDirectory)/msgraph-metadata
# publish metadata as an artifact
@@ -88,16 +96,16 @@ jobs:
sourceFolder: ${{ parameters.outputPath }}
contents: '**/$(Name).yaml'
targetFolder: '$(Build.ArtifactStagingDirectory)/$(Name)'
- displayName: Copy generated metadata
+ displayName: Copy generated OpenAPI yaml file to ArtifactStagingDirectory
+
- pwsh: |
./scripts/run-openapi-validation.ps1 -repoDirectory (Get-Location).Path -version "${{ parameters.endpoint }}" -platformName "$(Name)"
- displayName: ensure that OpenAPI docs can be parsed
+ displayName: Validate that OpenAPI docs can be parsed
workingDirectory: $(Build.SourcesDirectory)/msgraph-metadata
-
- job: publish_openapi
dependsOn: convert_openapi
- displayName: Publish
+ displayName: Publish OpenAPI files
## If there's new settings added please add them here too
templateContext:
inputs:
@@ -123,35 +131,61 @@ jobs:
displayName: checkout generator
fetchDepth: 1
persistCredentials: true
- # Copy files from the maxtrix artifacts to a single folder
+
+ # Note that msgraph-metadata repo has detached HEAD
+ - template: /.azure-pipelines/generation-templates/checkout-metadata.yml@self
+
+ - template: /.azure-pipelines/generation-templates/set-user-config.yml@self
+
+ # Copy OpenAPI files from the matrix artifacts to the ArtifactStagingDirectory
+ # This is used for library generation later
- task: CopyFiles@2
inputs:
sourceFolder: '$(Build.SourcesDirectory)/artifacts'
contents: '**/*.yaml'
targetFolder: '$(Build.ArtifactStagingDirectory)'
- displayName: Copy artifact metadata
- # Copy files from the target path where artifacts should be downloaded
+ displayName: Copy OpenAPI input artifacts to artifact staging directory
+
+ # Copy OpenAPI files from ArtifactStagingDirectory to local msgraph-metadata repo
- task: CopyFiles@2
inputs:
sourceFolder: '$(Build.ArtifactStagingDirectory)'
contents: '**/*.yaml'
targetFolder: '$(Build.SourcesDirectory)/msgraph-metadata/openapi/${{ parameters.endpoint }}'
- displayName: Copy downloaded metadata
+ overwrite: true
+ displayName: Copy OpenAPI files to local msgraph-metadata repo
- # publish metadata as an artifact
- - task: CopyFiles@2
- inputs:
- sourceFolder: ${{ parameters.outputPath }}
- contents: '**/*.yaml'
- targetFolder: '$(Build.ArtifactStagingDirectory)'
- displayName: Copy downloaded metadata
- - template: /.azure-pipelines/generation-templates/checkout-metadata.yml@self
- - template: /.azure-pipelines/generation-templates/set-user-config.yml@self
+ # Push changes to msgraph-metadata repo
- pwsh: '$(scriptsDirectory)/git-push-cleanmetadata.ps1'
-
- displayName: push clean ${{ parameters.endpoint }} OpenAPI description to msgraph-metadata repo
+ displayName: Publish ${{ parameters.endpoint }} OpenAPI description to msgraph-metadata repo
env:
+ CreateOpenAPIPR: True
EndpointVersion: ${{ parameters.endpoint }}
PublishChanges: $(publishChanges)
workingDirectory: '$(Build.SourcesDirectory)/msgraph-metadata'
enabled: true
+
+
+ # Create PR - note that this PR is not used for gating. It's just for discovery purposes.
+ # Library generation PRs will still be created based on the OpenAPI files.
+
+ - task: AzureKeyVault@2
+ displayName: "Azure Key Vault: Get Secrets"
+ inputs:
+ azureSubscription: "Federated AKV Managed Identity Connection"
+ KeyVaultName: akv-prod-eastus
+ SecretsFilter: "microsoft-graph-devx-bot-appid,microsoft-graph-devx-bot-privatekey"
+
+ - pwsh: '$(scriptsDirectory)/create-pull-request.ps1'
+ displayName: 'Create Pull Request for the generated OpenAPI files for msgraph-metadata'
+ env:
+ BaseBranch: master
+ GeneratePullRequest: true
+ GhAppId: $(microsoft-graph-devx-bot-appid)
+ GhAppKey: $(microsoft-graph-devx-bot-privatekey)
+ OverrideSkipCI: false
+ RepoName: 'microsoftgraph/msgraph-metadata'
+ ScriptsDirectory: $(scriptsDirectory)
+ # Version is intentionally left empty for OpenAPI PRs as versioning is not applicable in this context.
+ Version: ''
+ workingDirectory: '$(Build.SourcesDirectory)/msgraph-metadata'
\ No newline at end of file
diff --git a/.azure-pipelines/generation-templates/use-dotnet-sdk.yml b/.azure-pipelines/generation-templates/use-dotnet-sdk.yml
index 24246b344..700958ae7 100644
--- a/.azure-pipelines/generation-templates/use-dotnet-sdk.yml
+++ b/.azure-pipelines/generation-templates/use-dotnet-sdk.yml
@@ -5,7 +5,7 @@ parameters:
steps:
- task: UseDotNet@2
- displayName: 'Use .NET SDK'
+ displayName: 'Use .NET SDK ${{ parameters.version }}'
inputs:
packageType: sdk
version: ${{ parameters.version }}
diff --git a/scripts/create-pull-request.ps1 b/scripts/create-pull-request.ps1
index 6b4571f30..d8932c341 100644
--- a/scripts/create-pull-request.ps1
+++ b/scripts/create-pull-request.ps1
@@ -1,24 +1,27 @@
if (($env:OverrideSkipCI -eq $False) -and ($env:BUILD_REASON -eq 'Manual')) # Skip CI if manually running this pipeline.
{
- Write-Host "Skipping pull request creation due Skip CI." -ForegroundColor Green
+ Write-Host "Skipping pull request creation due Skip CI."
return;
}
if (($env:GeneratePullRequest -eq $False)) { # Skip CI if manually running this pipeline.
- Write-Host "Skipping pull request creation due this repository being disabled" -ForegroundColor Green
+ Write-Host "Skipping pull request creation due this repository being disabled"
return;
}
-# Special case for beta typings as it uses a non-conforming preview versioning.
+# Special case for beta typings as it uses a non-conforming preview versioning. Helps with triggering Release Please.
if ($env:RepoName.Contains("msgraph-beta-typescript-typings"))
{
- $title = "feat: generated $version models and request builders"
+ $title = "feat: generated $env:Version models and request builders"
+}
+elseif ($env:RepoName.Contains("msgraph-metadata")) # we are only generating OpenAPI PRs for the metadata repo
+{
+ $title = "Generated $env:Version OpenAPI descriptions"
}
else {
- $title = "Generated $version models and request builders"
+ $title = "Generated $env:Version models and request builders"
}
-$version = $env:Version
$body = ":bangbang:**_Important_**:bangbang:
Check for unexpected deletions or changes in this PR and ensure relevant CI checks are passing.
**Note:** This pull request was automatically created by Azure pipelines."
$baseBranchParameter = ""
@@ -30,10 +33,10 @@ if (![string]::IsNullOrEmpty($env:BaseBranch))
# The installed application is required to have the following permissions: read/write on pull requests/
$tokenGenerationScript = "$env:ScriptsDirectory\Generate-Github-Token.ps1"
$env:GITHUB_TOKEN = & $tokenGenerationScript -AppClientId $env:GhAppId -AppPrivateKeyContents $env:GhAppKey -Repository $env:RepoName
-Write-Host "Fetched Github Token for PR generation and set as environment variable." -ForegroundColor Green
+Write-Host "Fetched Github Token for PR generation and set as environment variable."
# No need to specify reviewers as code owners should be added automatically.
Invoke-Expression "gh auth login" # login to GitHub
Invoke-Expression "gh pr create -t ""$title"" -b ""$body"" $baseBranchParameter | Write-Host"
-Write-Host "Pull Request Created successfully." -ForegroundColor Green
\ No newline at end of file
+Write-Host "Pull Request Created successfully."
\ No newline at end of file
diff --git a/scripts/generate-open-api.ps1 b/scripts/generate-open-api.ps1
index a3634f115..1788e484b 100644
--- a/scripts/generate-open-api.ps1
+++ b/scripts/generate-open-api.ps1
@@ -21,6 +21,8 @@ param(
[parameter(Mandatory = $true)][String]$platformName
)
+Write-Host "Starting $endpointVersion OpenAPI generation for $platformName using generate-open-api.ps1"
+
$outputFile = Join-Path "./" "openapi" $endpointVersion "$platformName.yaml"
$oldOutputFile = "$outputFile.old"
$cleanVersion = $endpointVersion.Replace(".", "")
@@ -34,16 +36,16 @@ if($platformName -eq "openapi")
$fileName = "$baseFileName$endpointVersion.xml";
$inputFile = Join-Path "./" "clean_$($cleanVersion)_metadata" "$fileName"
-Write-Host "Settings: $settings"
-Write-Verbose "Generating OpenAPI description from $inputFile"
-Write-Verbose "Output file: $outputFile"
+Write-Host "`nSettings: $settings"
+Write-Host "Generating OpenAPI description from $inputFile"
+Write-Host "Output file: $outputFile"
if(Test-Path $outputFile)
{
- Write-Verbose "Removing existing output file"
+ Write-Host "`nRemoving existing output file"
if(Test-Path $oldOutputFile)
{
- Write-Verbose "Removing existing old output file"
+ Write-Host "Removing existing old output file: $oldOutputFile"
Remove-Item $oldOutputFile -Force
}
$oldFileName = Split-Path $outputFile -leaf
@@ -51,6 +53,7 @@ if(Test-Path $outputFile)
Rename-Item $outputFile $oldFileName
}
+Write-Host "`nGenerating OpenAPI description using hidi..."
$command = "hidi transform --csdl ""$inputFile"" --output ""$outputFile"" --settings-path ""$settings"" --version ""3.0"" --metadata-version ""$endpointVersion"" --log-level Information --format yaml"
Write-Host $command
@@ -62,16 +65,18 @@ try {
Set-Content $outputFile $updatedContent -NoNewline
if(Test-Path $oldOutputFile)
{
- Write-Verbose "Removing existing old output file"
+ Write-Host "`nRemoving existing old output file: $oldOutputFile"
Remove-Item $oldOutputFile -Force
}
+ Write-Host "Completed generating OpenAPI description using hidi"
} catch {
if(Test-Path $oldOutputFile)
{
- Write-Warning "Restoring old output file"
+ Write-Host "`nRestoring old output file: $oldOutputFile"
$originalFileName = Split-Path $outputFile -leaf
Rename-Item $oldOutputFile $originalFileName
}
Write-Error "Error generating OpenAPI description: $_"
throw $_
}
+
diff --git a/scripts/git-push-cleanmetadata.ps1 b/scripts/git-push-cleanmetadata.ps1
index cd33e51b1..a580d8a89 100644
--- a/scripts/git-push-cleanmetadata.ps1
+++ b/scripts/git-push-cleanmetadata.ps1
@@ -1,31 +1,85 @@
+# git-push-cleanmetadata.ps1
+# Copyright (c) Microsoft Corporation. All Rights Reserved. Licensed under the MIT License. See License in the
+# project root for license information.
+
+# Referenced by:
+# capture-metadata.yml
+# capture-openapi.yml
+
+# This script stashes any changes, checks out the latest master branch, applies the stashed changes, commits, and
+# pushes the changes back to the remote repository.
+
if ($env:PublishChanges -eq $False)
{
- Write-Host "Not publishing changes per the run parameter!" -ForegroundColor Green
+ Write-Host "Not publishing changes per the run parameter!"
return;
}
-Write-Host "About to add clean $env:EndpointVersion metadata file....."
+Write-Host "`n1. git status:"
+git status | Write-Host
-git fetch origin master
+Write-Host "`n2. Stash the update metadata files.....`n3. Running: git stash"
+git stash | Write-Host
+
+Write-Host "`n4. Fetching latest master branch to ensure we are up to date..."
+git fetch origin master | Write-Host
# checkout master to move from detached HEAD mode
-git switch master
+git switch master | Write-Host
+
+
+
+Write-Host "`n5. git status:"
+git status | Write-Host
+
+Write-Host "`n6. Apply stashed metadata files...`n7. Running: git stash pop"
+git stash pop | Write-Host
+
+Write-Host "`n8. git status:"
+git status | Write-Host
+
+$branch = "publish-open-api-files/$env:BUILD_BUILDID/$env:EndpointVersion"
+if ($env:CreateOpenAPIPR -eq $True)
+{
+ Write-Host "`n9. Create branch: $branch"
+ git checkout -B $branch | Write-Host
+}
+
+Write-Host "`n10. Staging clean $env:EndpointVersion metadata files.....`n11. Running: git add ."
git add . | Write-Host
+
+Write-Host "`n12. git status:"
+git status | Write-Host
+
+Write-Host "`n13. Attempting to commit clean $env:EndpointVersion metadata files....."
+
if ($env:BUILD_REASON -eq 'Manual') # Skip CI if manually running this pipeline.
{
- git commit -m "Update clean metadata file with $env:BUILD_BUILDID [skip ci]" | Write-Host
+ git commit -m "Update clean $env:EndpointVersion metadata file with $env:BUILD_BUILDID [skip ci]" | Write-Host
}
else
{
- git commit -m "Update clean metadata file with $env:BUILD_BUILDID" | Write-Host
+ git commit -m "Update clean $env:EndpointVersion metadata file with $env:BUILD_BUILDID" | Write-Host
}
-Write-Host "Added and commited cleaned $env:EndpointVersion metadata." -ForegroundColor Green
+Write-Host "`n14. git status:"
+git status | Write-Host
+
+if ($env:CreateOpenAPIPR -eq $True)
+{
+ Write-Host "`n15a. Pushing branch for PR creation"
-# sync branch before pushing
-# this is especially important while running v1 and beta in parallel
-# and one process goes out of sync because of the other's check-in
-git pull origin master --rebase
+ Write-Host "`n15b. Running: git push --set-upstream origin $branch"
+ git push --set-upstream origin $branch | Write-Host
+}
+else # original behavior: push to master
+{
+ Write-Host "`n15c. Running: git pull origin master --rebase..."
+ # sync branch before pushing
+ # this is especially important while running v1 and beta in parallel
+ # and one process goes out of sync because of the other's check-in
+ git pull origin master --rebase | Write-Host
-git push --set-upstream origin master | Write-Host
-Write-Host "Pushed the results of the build $env:BUILD_BUILDID to the master branch." -ForegroundColor Green
\ No newline at end of file
+ Write-Host "`n15d. Running: git push --set-upstream origin master ..."
+ git push --set-upstream origin master | Write-Host
+}
\ No newline at end of file