diff --git a/.azure-pipelines/generation-pipeline.yml b/.azure-pipelines/generation-pipeline.yml index 0d396af41..46e179ea7 100644 --- a/.azure-pipelines/generation-pipeline.yml +++ b/.azure-pipelines/generation-pipeline.yml @@ -3,8 +3,15 @@ trigger: none pr: none schedules: -- cron: '0 10 * * TUE' - displayName: Tuesday generation (PST 2am, EST 5am, EAT 3pm) + +# Minute: 0 → At the start of the hour. +# Hour: 10 → 10:00 AM UTC. +# Day of Month: 1-7,15-21 → On the 1st–7th and 15th–21st. +# Month: * → Every month. +# Weekday: TUE → Only on Tuesdays. +# This cron will run bi-weekly unless the month has 5 Tuesdays, in which case it will skip the 5th Tuesday (won't be bi-weekly that month). +- cron: '0 10 1-7,15-21 * TUE' + displayName: Bi-weekly Tuesday generation (1st and 3rd Tuesday, PST 2am, EST 5am, EAT 3pm) branches: include: - main @@ -291,10 +298,45 @@ extends: outputPath: $(cleanOpenAPIFileBetaOutputPath) cleanMetadataFolder: $(cleanOpenAPIFolderBeta) + # Approval stage for v1 OpenAPI generation. All subsequent stages depending on OpenAPI generation + # will be blocked until approval is granted. Any new stages depending on OpenAPI generation + # should also depend on this approval stage. + - stage: open_api_v1_approval + dependsOn: + - stage_v1_openapi + condition: eq(dependencies.stage_v1_openapi.result, 'Succeeded') + jobs: + - deployment: OpenApiApproval + displayName: 'OpenAPI v1.0 Generation Review' + environment: 'openapi-generation-review' + strategy: + runOnce: + deploy: + steps: + - script: echo "OpenAPI generation approved." + + # Approval stage for beta OpenAPI generation. All subsequent stages depending on OpenAPI generation + # will be blocked until approval is granted. Any new stages depending on OpenAPI generation + # should also depend on this approval stage. + - stage: open_api_beta_approval + dependsOn: + - stage_beta_openapi + condition: eq(dependencies.stage_beta_openapi.result, 'Succeeded') + jobs: + - deployment: OpenApiApproval + displayName: 'OpenAPI Beta Generation Review' + environment: 'openapi-generation-review' + strategy: + runOnce: + deploy: + steps: + - script: echo "OpenAPI generation approved." + - stage: stage_csharp_v1_kiota dependsOn: - stage_build_and_publish_kiota - stage_v1_openapi + - open_api_v1_approval condition: | and ( @@ -336,6 +378,7 @@ extends: dependsOn: - stage_build_and_publish_kiota - stage_beta_openapi + - open_api_beta_approval condition: | and ( @@ -377,6 +420,7 @@ extends: dependsOn: - stage_build_and_publish_kiota - stage_v1_openapi + - open_api_v1_approval condition: | and ( @@ -416,6 +460,7 @@ extends: dependsOn: - stage_build_and_publish_kiota - stage_beta_openapi + - open_api_beta_approval condition: | and ( @@ -455,6 +500,7 @@ extends: dependsOn: - stage_build_and_publish_kiota - stage_v1_openapi + - open_api_v1_approval condition: | and ( @@ -496,6 +542,7 @@ extends: dependsOn: - stage_build_and_publish_kiota - stage_beta_openapi + - open_api_beta_approval condition: | and ( @@ -537,6 +584,7 @@ extends: dependsOn: - stage_build_and_publish_kiota - stage_beta_openapi + - open_api_beta_approval condition: | and ( @@ -578,6 +626,7 @@ extends: dependsOn: - stage_build_and_publish_kiota - stage_v1_openapi + - open_api_v1_approval condition: | and ( @@ -693,6 +742,7 @@ extends: dependsOn: - stage_build_and_publish_kiota - stage_v1_openapi + - open_api_v1_approval condition: | and ( @@ -735,6 +785,7 @@ extends: dependsOn: - stage_build_and_publish_kiota - stage_beta_openapi + - open_api_beta_approval condition: | and ( @@ -777,6 +828,7 @@ extends: dependsOn: - stage_build_and_publish_kiota - stage_v1_openapi + - open_api_v1_approval condition: | and ( @@ -817,6 +869,7 @@ extends: dependsOn: - stage_build_and_publish_kiota - stage_beta_openapi + - open_api_beta_approval condition: | and ( diff --git a/.azure-pipelines/generation-templates/capture-metadata.yml b/.azure-pipelines/generation-templates/capture-metadata.yml index 96637da28..4a6fb2377 100644 --- a/.azure-pipelines/generation-templates/capture-metadata.yml +++ b/.azure-pipelines/generation-templates/capture-metadata.yml @@ -166,9 +166,9 @@ steps: # Checkin clean metadata into metadata repo or make it an artifact. - pwsh: '$(scriptsDirectory)/git-push-cleanmetadata.ps1' - - displayName: push clean ${{ parameters.endpoint }} metadata to msgraph-metadata repo + displayName: push clean ${{ parameters.endpoint }} CSDL metadata to msgraph-metadata repo env: + CreateOpenAPIPR: False EndpointVersion: ${{ parameters.endpoint }} PublishChanges: $(publishChanges) workingDirectory: '$(Build.SourcesDirectory)/msgraph-metadata' diff --git a/.azure-pipelines/generation-templates/capture-openapi.yml b/.azure-pipelines/generation-templates/capture-openapi.yml index 617931c8d..79218a552 100644 --- a/.azure-pipelines/generation-templates/capture-openapi.yml +++ b/.azure-pipelines/generation-templates/capture-openapi.yml @@ -1,3 +1,4 @@ +# capture-openapi.yml # Copyright (c) Microsoft Corporation. All Rights Reserved. Licensed under the MIT License. See License in the project root for license information. # The following template creates multiple artifacts from a matrix, this is the intended behavior to follow the same workflow before the governance migration @@ -58,13 +59,12 @@ jobs: persistCredentials: true - template: /.azure-pipelines/generation-templates/checkout-metadata.yml@self - # required for the hidi to run - template: /.azure-pipelines/generation-templates/use-dotnet-sdk.yml@self parameters: version: '8.x' -# required for the hidi installation validation + # required for the hidi installation validation - template: /.azure-pipelines/generation-templates/use-dotnet-sdk.yml@self parameters: version: '9.x' @@ -78,8 +78,13 @@ jobs: displayName: ensure the generation happens from master latest workingDirectory: $(Build.SourcesDirectory)/msgraph-metadata - - pwsh: '$(scriptsDirectory)/generate-open-api.ps1 -endpointVersion ${{ parameters.endpoint }} -settings "$(conversionSettingsDirectory)/$(File)" -platformName "$(Name)"' - displayName: 'update ${{ parameters.endpoint }} open API description' + - pwsh: | + Write-Host "`ngit status before generation:" + git status + $(scriptsDirectory)/generate-open-api.ps1 -endpointVersion ${{ parameters.endpoint }} -settings "$(conversionSettingsDirectory)/$(File)" -platformName "$(Name)" + Write-Host "`ngit status after generation:" + git status + displayName: 'generate ${{ parameters.endpoint }} open API description' workingDirectory: $(Build.SourcesDirectory)/msgraph-metadata # publish metadata as an artifact @@ -88,16 +93,16 @@ jobs: sourceFolder: ${{ parameters.outputPath }} contents: '**/$(Name).yaml' targetFolder: '$(Build.ArtifactStagingDirectory)/$(Name)' - displayName: Copy generated metadata + displayName: Copy generated OpenAPI yaml file to ArtifactStagingDirectory + - pwsh: | ./scripts/run-openapi-validation.ps1 -repoDirectory (Get-Location).Path -version "${{ parameters.endpoint }}" -platformName "$(Name)" - displayName: ensure that OpenAPI docs can be parsed + displayName: Validate that OpenAPI docs can be parsed workingDirectory: $(Build.SourcesDirectory)/msgraph-metadata - - job: publish_openapi dependsOn: convert_openapi - displayName: Publish + displayName: Publish OpenAPI files ## If there's new settings added please add them here too templateContext: inputs: @@ -123,35 +128,61 @@ jobs: displayName: checkout generator fetchDepth: 1 persistCredentials: true - # Copy files from the maxtrix artifacts to a single folder + + # Note that msgraph-metadata repo has detached HEAD + - template: /.azure-pipelines/generation-templates/checkout-metadata.yml@self + + - template: /.azure-pipelines/generation-templates/set-user-config.yml@self + + # Copy OpenAPI files from the matrix artifacts to the ArtifactStagingDirectory + # This is used for library generation later - task: CopyFiles@2 inputs: sourceFolder: '$(Build.SourcesDirectory)/artifacts' contents: '**/*.yaml' targetFolder: '$(Build.ArtifactStagingDirectory)' - displayName: Copy artifact metadata - # Copy files from the target path where artifacts should be downloaded + displayName: Copy OpenAPI input artifacts to artifact staging directory + + # Copy OpenAPI files from ArtifactStagingDirectory to local msgraph-metadata repo - task: CopyFiles@2 inputs: sourceFolder: '$(Build.ArtifactStagingDirectory)' contents: '**/*.yaml' targetFolder: '$(Build.SourcesDirectory)/msgraph-metadata/openapi/${{ parameters.endpoint }}' - displayName: Copy downloaded metadata + overwrite: true + displayName: Copy OpenAPI files to local msgraph-metadata repo - # publish metadata as an artifact - - task: CopyFiles@2 - inputs: - sourceFolder: ${{ parameters.outputPath }} - contents: '**/*.yaml' - targetFolder: '$(Build.ArtifactStagingDirectory)' - displayName: Copy downloaded metadata - - template: /.azure-pipelines/generation-templates/checkout-metadata.yml@self - - template: /.azure-pipelines/generation-templates/set-user-config.yml@self + # Push changes to msgraph-metadata repo - pwsh: '$(scriptsDirectory)/git-push-cleanmetadata.ps1' - - displayName: push clean ${{ parameters.endpoint }} OpenAPI description to msgraph-metadata repo + displayName: Publish ${{ parameters.endpoint }} OpenAPI description to msgraph-metadata repo env: + CreateOpenAPIPR: True EndpointVersion: ${{ parameters.endpoint }} PublishChanges: $(publishChanges) workingDirectory: '$(Build.SourcesDirectory)/msgraph-metadata' enabled: true + + + # Create PR - note that this PR is not used for gating. It's just for discovery purposes. + # Library generation PRs will still be created based on the OpenAPI files. + + - task: AzureKeyVault@2 + displayName: "Azure Key Vault: Get Secrets" + inputs: + azureSubscription: "Federated AKV Managed Identity Connection" + KeyVaultName: akv-prod-eastus + SecretsFilter: "microsoft-graph-devx-bot-appid,microsoft-graph-devx-bot-privatekey" + + - pwsh: '$(scriptsDirectory)/create-pull-request.ps1' + displayName: 'Create Pull Request for the generated OpenAPI files for msgraph-metadata' + env: + BaseBranch: master + GeneratePullRequest: true + GhAppId: $(microsoft-graph-devx-bot-appid) + GhAppKey: $(microsoft-graph-devx-bot-privatekey) + OverrideSkipCI: false + RepoName: 'microsoftgraph/msgraph-metadata' + ScriptsDirectory: $(scriptsDirectory) + # Version is intentionally left empty for OpenAPI PRs as versioning is not applicable in this context. + Version: '' + workingDirectory: '$(Build.SourcesDirectory)/msgraph-metadata' \ No newline at end of file diff --git a/.azure-pipelines/generation-templates/use-dotnet-sdk.yml b/.azure-pipelines/generation-templates/use-dotnet-sdk.yml index 24246b344..700958ae7 100644 --- a/.azure-pipelines/generation-templates/use-dotnet-sdk.yml +++ b/.azure-pipelines/generation-templates/use-dotnet-sdk.yml @@ -5,7 +5,7 @@ parameters: steps: - task: UseDotNet@2 - displayName: 'Use .NET SDK' + displayName: 'Use .NET SDK ${{ parameters.version }}' inputs: packageType: sdk version: ${{ parameters.version }} diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index 7a6aac7e4..f1882995e 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -52,7 +52,7 @@ jobs: # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL - uses: github/codeql-action/init@v3 + uses: github/codeql-action/init@v4 with: languages: ${{ matrix.language }} # If you wish to specify custom queries, you can do so here or in a config file. @@ -66,7 +66,7 @@ jobs: # Autobuild attempts to build any compiled languages (C/C++, C#, Go, Java, or Swift). # If this step fails, then you should remove it and run the build manually (see below) - name: Autobuild - uses: github/codeql-action/autobuild@v3 + uses: github/codeql-action/autobuild@v4 with: working-directory: ${{ github.workspace }}/src/Typewriter @@ -81,6 +81,6 @@ jobs: # ./location_of_script_within_repo/buildscript.sh - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@v3 + uses: github/codeql-action/analyze@v4 with: category: "/language:${{matrix.language}}" diff --git a/scripts/create-pull-request.ps1 b/scripts/create-pull-request.ps1 index 6b4571f30..d8932c341 100644 --- a/scripts/create-pull-request.ps1 +++ b/scripts/create-pull-request.ps1 @@ -1,24 +1,27 @@ if (($env:OverrideSkipCI -eq $False) -and ($env:BUILD_REASON -eq 'Manual')) # Skip CI if manually running this pipeline. { - Write-Host "Skipping pull request creation due Skip CI." -ForegroundColor Green + Write-Host "Skipping pull request creation due Skip CI." return; } if (($env:GeneratePullRequest -eq $False)) { # Skip CI if manually running this pipeline. - Write-Host "Skipping pull request creation due this repository being disabled" -ForegroundColor Green + Write-Host "Skipping pull request creation due this repository being disabled" return; } -# Special case for beta typings as it uses a non-conforming preview versioning. +# Special case for beta typings as it uses a non-conforming preview versioning. Helps with triggering Release Please. if ($env:RepoName.Contains("msgraph-beta-typescript-typings")) { - $title = "feat: generated $version models and request builders" + $title = "feat: generated $env:Version models and request builders" +} +elseif ($env:RepoName.Contains("msgraph-metadata")) # we are only generating OpenAPI PRs for the metadata repo +{ + $title = "Generated $env:Version OpenAPI descriptions" } else { - $title = "Generated $version models and request builders" + $title = "Generated $env:Version models and request builders" } -$version = $env:Version $body = ":bangbang:**_Important_**:bangbang:
Check for unexpected deletions or changes in this PR and ensure relevant CI checks are passing.

**Note:** This pull request was automatically created by Azure pipelines." $baseBranchParameter = "" @@ -30,10 +33,10 @@ if (![string]::IsNullOrEmpty($env:BaseBranch)) # The installed application is required to have the following permissions: read/write on pull requests/ $tokenGenerationScript = "$env:ScriptsDirectory\Generate-Github-Token.ps1" $env:GITHUB_TOKEN = & $tokenGenerationScript -AppClientId $env:GhAppId -AppPrivateKeyContents $env:GhAppKey -Repository $env:RepoName -Write-Host "Fetched Github Token for PR generation and set as environment variable." -ForegroundColor Green +Write-Host "Fetched Github Token for PR generation and set as environment variable." # No need to specify reviewers as code owners should be added automatically. Invoke-Expression "gh auth login" # login to GitHub Invoke-Expression "gh pr create -t ""$title"" -b ""$body"" $baseBranchParameter | Write-Host" -Write-Host "Pull Request Created successfully." -ForegroundColor Green \ No newline at end of file +Write-Host "Pull Request Created successfully." \ No newline at end of file diff --git a/scripts/generate-open-api.ps1 b/scripts/generate-open-api.ps1 index a3634f115..1788e484b 100644 --- a/scripts/generate-open-api.ps1 +++ b/scripts/generate-open-api.ps1 @@ -21,6 +21,8 @@ param( [parameter(Mandatory = $true)][String]$platformName ) +Write-Host "Starting $endpointVersion OpenAPI generation for $platformName using generate-open-api.ps1" + $outputFile = Join-Path "./" "openapi" $endpointVersion "$platformName.yaml" $oldOutputFile = "$outputFile.old" $cleanVersion = $endpointVersion.Replace(".", "") @@ -34,16 +36,16 @@ if($platformName -eq "openapi") $fileName = "$baseFileName$endpointVersion.xml"; $inputFile = Join-Path "./" "clean_$($cleanVersion)_metadata" "$fileName" -Write-Host "Settings: $settings" -Write-Verbose "Generating OpenAPI description from $inputFile" -Write-Verbose "Output file: $outputFile" +Write-Host "`nSettings: $settings" +Write-Host "Generating OpenAPI description from $inputFile" +Write-Host "Output file: $outputFile" if(Test-Path $outputFile) { - Write-Verbose "Removing existing output file" + Write-Host "`nRemoving existing output file" if(Test-Path $oldOutputFile) { - Write-Verbose "Removing existing old output file" + Write-Host "Removing existing old output file: $oldOutputFile" Remove-Item $oldOutputFile -Force } $oldFileName = Split-Path $outputFile -leaf @@ -51,6 +53,7 @@ if(Test-Path $outputFile) Rename-Item $outputFile $oldFileName } +Write-Host "`nGenerating OpenAPI description using hidi..." $command = "hidi transform --csdl ""$inputFile"" --output ""$outputFile"" --settings-path ""$settings"" --version ""3.0"" --metadata-version ""$endpointVersion"" --log-level Information --format yaml" Write-Host $command @@ -62,16 +65,18 @@ try { Set-Content $outputFile $updatedContent -NoNewline if(Test-Path $oldOutputFile) { - Write-Verbose "Removing existing old output file" + Write-Host "`nRemoving existing old output file: $oldOutputFile" Remove-Item $oldOutputFile -Force } + Write-Host "Completed generating OpenAPI description using hidi" } catch { if(Test-Path $oldOutputFile) { - Write-Warning "Restoring old output file" + Write-Host "`nRestoring old output file: $oldOutputFile" $originalFileName = Split-Path $outputFile -leaf Rename-Item $oldOutputFile $originalFileName } Write-Error "Error generating OpenAPI description: $_" throw $_ } + diff --git a/scripts/git-push-cleanmetadata.ps1 b/scripts/git-push-cleanmetadata.ps1 index cd33e51b1..a580d8a89 100644 --- a/scripts/git-push-cleanmetadata.ps1 +++ b/scripts/git-push-cleanmetadata.ps1 @@ -1,31 +1,85 @@ +# git-push-cleanmetadata.ps1 +# Copyright (c) Microsoft Corporation. All Rights Reserved. Licensed under the MIT License. See License in the +# project root for license information. + +# Referenced by: +# capture-metadata.yml +# capture-openapi.yml + +# This script stashes any changes, checks out the latest master branch, applies the stashed changes, commits, and +# pushes the changes back to the remote repository. + if ($env:PublishChanges -eq $False) { - Write-Host "Not publishing changes per the run parameter!" -ForegroundColor Green + Write-Host "Not publishing changes per the run parameter!" return; } -Write-Host "About to add clean $env:EndpointVersion metadata file....." +Write-Host "`n1. git status:" +git status | Write-Host -git fetch origin master +Write-Host "`n2. Stash the update metadata files.....`n3. Running: git stash" +git stash | Write-Host + +Write-Host "`n4. Fetching latest master branch to ensure we are up to date..." +git fetch origin master | Write-Host # checkout master to move from detached HEAD mode -git switch master +git switch master | Write-Host + + + +Write-Host "`n5. git status:" +git status | Write-Host + +Write-Host "`n6. Apply stashed metadata files...`n7. Running: git stash pop" +git stash pop | Write-Host + +Write-Host "`n8. git status:" +git status | Write-Host + +$branch = "publish-open-api-files/$env:BUILD_BUILDID/$env:EndpointVersion" +if ($env:CreateOpenAPIPR -eq $True) +{ + Write-Host "`n9. Create branch: $branch" + git checkout -B $branch | Write-Host +} + +Write-Host "`n10. Staging clean $env:EndpointVersion metadata files.....`n11. Running: git add ." git add . | Write-Host + +Write-Host "`n12. git status:" +git status | Write-Host + +Write-Host "`n13. Attempting to commit clean $env:EndpointVersion metadata files....." + if ($env:BUILD_REASON -eq 'Manual') # Skip CI if manually running this pipeline. { - git commit -m "Update clean metadata file with $env:BUILD_BUILDID [skip ci]" | Write-Host + git commit -m "Update clean $env:EndpointVersion metadata file with $env:BUILD_BUILDID [skip ci]" | Write-Host } else { - git commit -m "Update clean metadata file with $env:BUILD_BUILDID" | Write-Host + git commit -m "Update clean $env:EndpointVersion metadata file with $env:BUILD_BUILDID" | Write-Host } -Write-Host "Added and commited cleaned $env:EndpointVersion metadata." -ForegroundColor Green +Write-Host "`n14. git status:" +git status | Write-Host + +if ($env:CreateOpenAPIPR -eq $True) +{ + Write-Host "`n15a. Pushing branch for PR creation" -# sync branch before pushing -# this is especially important while running v1 and beta in parallel -# and one process goes out of sync because of the other's check-in -git pull origin master --rebase + Write-Host "`n15b. Running: git push --set-upstream origin $branch" + git push --set-upstream origin $branch | Write-Host +} +else # original behavior: push to master +{ + Write-Host "`n15c. Running: git pull origin master --rebase..." + # sync branch before pushing + # this is especially important while running v1 and beta in parallel + # and one process goes out of sync because of the other's check-in + git pull origin master --rebase | Write-Host -git push --set-upstream origin master | Write-Host -Write-Host "Pushed the results of the build $env:BUILD_BUILDID to the master branch." -ForegroundColor Green \ No newline at end of file + Write-Host "`n15d. Running: git push --set-upstream origin master ..." + git push --set-upstream origin master | Write-Host +} \ No newline at end of file