diff --git a/.azure-pipelines/.vsts.release.yml b/.azure-pipelines/.vsts.release.yml new file mode 100644 index 00000000000..f6aebb06fdc --- /dev/null +++ b/.azure-pipelines/.vsts.release.yml @@ -0,0 +1,220 @@ +# This Yaml Document has been converted by ESAI Yaml Pipeline Conversion Tool. +# This pipeline will be extended to the OneESPT template + +trigger: none + +resources: + repositories: + - repository: 1ESPipelineTemplates + type: git + name: 1ESPipelineTemplates/1ESPipelineTemplates + ref: refs/tags/release + +parameters: +- name: task_deployment + displayName: Choose deployment options + type: string + default: 'Deploy task of Sprint' + values: + - 'Deploy all Tasks' + - 'Deploy task of Sprint' + - 'Deploy Hotfix' +- name: task_name + displayName: | + To 'Deploy Hotfix' provide task names (e.g. BashV3,AzureCLIV1,PowerShellV2) else leave to default + type: string + default: 'TaskNameVN' +- name: enableCodeQL + displayName: Enable CodeQL for run + type: boolean + default: false +- name: includeLocalPackagesBuildConfig + displayName: Flag to update LocalPackages buildconfig (for testing, this will be made default later) + type: boolean + default: false # note: keep in sync with ci\ci-test-tasks\canary-tests-v2.yml +- name: skip_publish + displayName: skipPublish (Not for production use) + type: boolean + default: false +- name: publishToDistributedTaskTest + displayName: Publish to test feed (DistributedTasks-test), for infrastucture testing + type: boolean + default: false +- name: dryRun + displayName: Dry Run (no push, no PR, no feed publish) + type: boolean + default: false + +variables: +- name: currentDate + value: $[ format('{0:yyyy}{0:MM}{0:dd}', pipeline.startTime) ] +- name: taskName + value: ${{ parameters.task_name }} +- name: taskNameIsSet + value: ${{ not(or(eq(parameters.task_deployment, 'Deploy all Tasks'), eq(parameters.task_deployment, 'Deploy task of Sprint'))) }} +- name: runCodeQl + value: ${{ eq(parameters.enableCodeQL, true) }} +- name: system.debug + value: true +- name: includeLocalPackagesBuildConfigParameter + ${{ if eq(parameters.includeLocalPackagesBuildConfig, true) }}: + value: '--includeLocalPackagesBuildConfig' + ${{ else }}: + value: '' +- name: IncludeLocalPackagesBuildConfigTest + ${{ if eq(parameters.includeLocalPackagesBuildConfig, true) }}: + value: '1' + ${{ else }}: + value: '' +- name: tasksSkipPublish + ${{ if eq(parameters.skip_publish, true) }}: + value: 'true' + ${{ else }}: + value: 'false' +- name: DEPLOY_ALL_TASKSVAR + ${{ if eq(parameters.task_deployment,'Deploy all Tasks') }}: + value: 'true' + ${{ else }}: + value: 'false' +- name: isDryRun + ${{ if eq(parameters.dryRun, true) }}: + value: 'true' + ${{ else }}: + value: 'false' + +extends: + template: v1/1ES.Official.PipelineTemplate.yml@1ESPipelineTemplates + parameters: + settings: + skipBuildTagsForGitHubPullRequests: true + featureFlags: + autoBaseline: false + sdl: + sbom: + enabled: true + baseline: + baselineSet: default + baselineFile: $(Build.SourcesDirectory)/.gdn/.gdnbaselines + sourceAnalysisPool: + name: 1ES-ABTT-Shared-Pool + image: abtt-windows-2022 + os: windows + sourceRepositoriesToScan: + exclude: + - repository: AzureDevOps + - repository: ConfigChange + customBuildTags: + - ES365AIMigrationTooling + stages: + - stage: main + jobs: + # All tasks on Windows + - job: build_all_windows + displayName: Build all tasks (Windows) + condition: eq(variables.os, 'Windows_NT') + timeoutInMinutes: 1440 # AntiMalware takes 3 hours to scan tasks.zip + pool: + name: 1ES-ABTT-Shared-Pool + image: abtt-windows-2022 + os: windows + templateContext: + outputs: + - output: buildArtifacts + displayName: 'Publish package artifact' + condition: > + and( + succeeded(), + ne(variables['build.reason'], 'PullRequest'), + ne(variables['numTasks'], 0) + ) + PathtoPublish: _package/tasks.zip + ArtifactName: package + sbomBuildDropPath: $(Build.SourcesDirectory)/_package + steps: + - template: /ci/build-all-steps.yml@self + parameters: + os: Windows_NT + + # Publish + - job: publish + displayName: Publish Nuget Artifacts + timeoutInMinutes: 360 + dependsOn: + - build_all_windows + condition: and(succeeded(), ne(variables['build.reason'], 'PullRequest'), eq(variables['tasksSkipPublish'], 'false')) + pool: + name: 1ES-ABTT-Shared-Pool + image: abtt-windows-2022 + os: windows + templateContext: + outputs: + - output: buildArtifacts + displayName: 'Publish per task NuGet package artifact' + PathtoPublish: _package/nuget-packages + ArtifactName: IndividualNuGetPackages + steps: + - template: /ci/publish-steps.yml@self + + # Courtesy Push + - job: courtesy_push + displayName: Courtesy Push + timeoutInMinutes: 180 + pool: + name: 1ES-ABTT-Shared-Pool + image: abtt-windows-2022 + os: windows + dependsOn: + - publish + condition: | + and( + succeeded(), + eq(variables['COURTESY_PUSH'], 'true'), + or( + eq(variables['build.reason'], 'Schedule'), + and( + eq(variables['build.reason'], 'Manual'), + eq(variables['FORCE_COURTESY_PUSH'], 'true') + ) + ) + ) + templateContext: + outputs: + - output: nuget + packagesToPush: '$(Build.SourcesDirectory)/IndividualNugetPackagesDownloaded/IndividualNugetPackages/*/*.nupkg' + packageParentPath: '$(Build.SourcesDirectory)' + ${{ if or(eq(parameters.dryRun, true),eq(parameters.publishToDistributedTaskTest, true)) }}: + publishVstsFeed: 'c86767d8-af79-4303-a7e6-21da0ba435e2/9d34d871-8032-4e10-a34a-c7a01e125865' + ${{ else }}: + publishVstsFeed: 'c86767d8-af79-4303-a7e6-21da0ba435e2/e10d0795-57cd-4d7f-904e-5f39703cb096' + nuGetFeedType: internal + displayName: Push Nuget package + allowPackageConflicts: $(COURTESY_PUSH) + steps: + - checkout: self + - template: /ci/courtesy-push.yml@self + + - job: build_all_tasks_for_deployments + displayName: Build all tasks for deployments (Windows) + dependsOn: + - courtesy_push + - build_all_windows + condition: succeeded() + timeoutInMinutes: 360 + pool: + name: 1ES-ABTT-Shared-Pool + image: abtt-windows-2022 + os: windows + variables: + task_pattern: $[dependencies.build_all_windows.outputs['getTaskPattern.task_pattern']] + templateContext: + outputs: + - output: buildArtifacts + displayName: 'Publish package artifact' + PathtoPublish: _package/tasks.zip + sbomBuildDropPath: $(Build.SourcesDirectory)/_package + ArtifactName: allTasks + steps: + - template: /ci/build-all-tasks.yml@self + parameters: + deploy_all_tasks: ${{ eq(parameters.task_deployment, 'Deploy all Tasks') }} + \ No newline at end of file diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 89b8e61e92d..28d794d9795 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -12,77 +12,25 @@ resources: name: 1ESPipelineTemplates/1ESPipelineTemplates ref: refs/tags/release -parameters: -- name: task_deployment - displayName: Choose deployment options - type: string - default: 'Deploy task of Sprint' - values: - - 'Deploy all Tasks' - - 'Deploy task of Sprint' - - 'Deploy Hotfix' -- name: task_name - displayName: | - To 'Deploy Hotfix' provide task names (e.g. BashV3,AzureCLIV1,PowerShellV2) else leave to default - type: string - default: 'TaskNameVN' -- name: enableCodeQL - displayName: Enable CodeQL for run - type: boolean - default: false -- name: includeLocalPackagesBuildConfig - displayName: Flag to update LocalPackages buildconfig (for testing, this will be made default later) - type: boolean - default: false # note: keep in sync with ci\ci-test-tasks\canary-tests-v2.yml -- name: skip_publish - displayName: skipPublish (Not for production use) - type: boolean - default: false -- name: publishToDistributedTaskTest - displayName: Publish to test feed (DistributedTasks-test), for infrastucture testing - type: boolean - default: false -- name: dryRun - displayName: Dry Run (no push, no PR, no feed publish) - type: boolean - default: false - variables: - name: currentDate value: $[ format('{0:yyyy}{0:MM}{0:dd}', pipeline.startTime) ] - name: taskName - value: ${{ parameters.task_name }} + value: 'TaskNameVN' - name: taskNameIsSet - value: ${{ not(or(eq(parameters.task_deployment, 'Deploy all Tasks'), eq(parameters.task_deployment, 'Deploy task of Sprint'))) }} + value: false - name: runCodeQl - value: ${{ eq(parameters.enableCodeQL, true) }} + value: false - name: system.debug value: true - name: includeLocalPackagesBuildConfigParameter - ${{ if eq(parameters.includeLocalPackagesBuildConfig, true) }}: - value: '--includeLocalPackagesBuildConfig' - ${{ else }}: - value: '' + value: '' - name: IncludeLocalPackagesBuildConfigTest - ${{ if eq(parameters.includeLocalPackagesBuildConfig, true) }}: - value: '1' - ${{ else }}: - value: '' -- name: tasksSkipPublish - ${{ if eq(parameters.skip_publish, true) }}: - value: 'true' - ${{ else }}: - value: 'false' + value: '' - name: DEPLOY_ALL_TASKSVAR - ${{ if eq(parameters.task_deployment,'Deploy all Tasks') }}: - value: 'true' - ${{ else }}: - value: 'false' + value: 'false' - name: isDryRun - ${{ if eq(parameters.dryRun, true) }}: - value: 'true' - ${{ else }}: - value: 'false' + value: 'false' extends: template: v1/1ES.Official.PipelineTemplate.yml@1ESPipelineTemplates @@ -137,89 +85,6 @@ extends: parameters: os: Windows_NT - # Publish - - job: publish - displayName: Publish Nuget Artifacts - timeoutInMinutes: 360 - dependsOn: - - build_all_windows - condition: and(succeeded(), ne(variables['build.reason'], 'PullRequest'), eq(variables['tasksSkipPublish'], 'false')) - pool: - name: 1ES-ABTT-Shared-Pool - image: abtt-windows-2022 - os: windows - templateContext: - outputs: - - output: buildArtifacts - displayName: 'Publish per task NuGet package artifact' - PathtoPublish: _package/nuget-packages - ArtifactName: IndividualNuGetPackages - steps: - - template: /ci/publish-steps.yml@self - - # Courtesy Push - - job: courtesy_push - displayName: Courtesy Push - timeoutInMinutes: 180 - pool: - name: 1ES-ABTT-Shared-Pool - image: abtt-windows-2022 - os: windows - dependsOn: - - publish - condition: | - and( - succeeded(), - eq(variables['COURTESY_PUSH'], 'true'), - or( - eq(variables['build.reason'], 'Schedule'), - and( - eq(variables['build.reason'], 'Manual'), - eq(variables['FORCE_COURTESY_PUSH'], 'true') - ) - ) - ) - templateContext: - outputs: - - output: nuget - packagesToPush: '$(Build.SourcesDirectory)/IndividualNugetPackagesDownloaded/IndividualNugetPackages/*/*.nupkg' - packageParentPath: '$(Build.SourcesDirectory)' - ${{ if or(eq(parameters.dryRun, true),eq(parameters.publishToDistributedTaskTest, true)) }}: - publishVstsFeed: 'c86767d8-af79-4303-a7e6-21da0ba435e2/9d34d871-8032-4e10-a34a-c7a01e125865' - ${{ else }}: - publishVstsFeed: 'c86767d8-af79-4303-a7e6-21da0ba435e2/e10d0795-57cd-4d7f-904e-5f39703cb096' - nuGetFeedType: internal - displayName: Push Nuget package - allowPackageConflicts: $(COURTESY_PUSH) - steps: - - checkout: self - - template: /ci/courtesy-push.yml@self - - - job: build_all_tasks_for_deployments - displayName: Build all tasks for deployments (Windows) - dependsOn: - - courtesy_push - - build_all_windows - condition: succeeded() - timeoutInMinutes: 360 - pool: - name: 1ES-ABTT-Shared-Pool - image: abtt-windows-2022 - os: windows - variables: - task_pattern: $[dependencies.build_all_windows.outputs['getTaskPattern.task_pattern']] - templateContext: - outputs: - - output: buildArtifacts - displayName: 'Publish package artifact' - PathtoPublish: _package/tasks.zip - sbomBuildDropPath: $(Build.SourcesDirectory)/_package - ArtifactName: allTasks - steps: - - template: /ci/build-all-tasks.yml@self - parameters: - deploy_all_tasks: ${{ eq(parameters.task_deployment, 'Deploy all Tasks') }} - # All tasks on Linux - job: build_all_linux displayName: Build all tasks (Linux) diff --git a/ci/build-single-ado-pr-steps.yml b/ci/build-single-ado-pr-steps.yml index a81f5bbdd8d..13fdf82e752 100644 --- a/ci/build-single-ado-pr-steps.yml +++ b/ci/build-single-ado-pr-steps.yml @@ -25,7 +25,7 @@ steps: artifactName: IndividualNugetPackages downloadPath: $(Build.SourcesDirectory)/IndividualNugetPackagesDownloaded displayName: Download Artifact - + - powershell: ./ci/set-sprint-variables.ps1 displayName: Set currentSprint variables @@ -40,7 +40,7 @@ steps: - script: node ci\courtesy-push\courtesy-push.js $(Build.SourcesDirectory)/IndividualNugetPackagesDownloaded\IndividualNugetPackages\unified_deps.xml displayName: Update unified deps and create branch env: - TOKEN: $(System.AccessToken) + TOKEN: $(AzDo_PAT) BRANCH_NAME: $(branchName) DRYRUN: $(isDryRun) USERNAME: $(username) \ No newline at end of file diff --git a/ci/courtesy-push.yml b/ci/courtesy-push.yml index b4c25a6b671..5c5ee1df90e 100644 --- a/ci/courtesy-push.yml +++ b/ci/courtesy-push.yml @@ -24,11 +24,12 @@ steps: displayName: Set currentDate variable - template: /ci/generate-branch-name.yml@self +- template: /ci/get-AzDo-pat.yml@self - script: node $(Build.SourcesDirectory)/ci/courtesy-push/courtesy-push.js $(Build.SourcesDirectory)/IndividualNugetPackagesDownloaded/IndividualNugetPackages/unified_deps.xml displayName: Update unified deps and create branch env: - TOKEN: $(System.AccessToken) + TOKEN: $(AzDo_PAT) BRANCH_NAME: $(branchName) DRYRUN: $(isDryRun) USERNAME: $(username) diff --git a/ci/courtesy-push/courtesy-push.js b/ci/courtesy-push/courtesy-push.js index 033fc419e9e..de71741930c 100644 --- a/ci/courtesy-push/courtesy-push.js +++ b/ci/courtesy-push/courtesy-push.js @@ -9,7 +9,7 @@ const msPrefix = 'Mseng.MS.TF.DistributedTask.Tasks.'; // Git configuration const GIT = 'git'; -const token = process.env.PAT || process.env.TOKEN; +const token = process.env.TOKEN; const orgUrl= 'dev.azure.com/mseng' const project = 'AzureDevOps'; const repo = 'AzureDevOps'; @@ -133,7 +133,7 @@ async function createPullRequest(sourceBranch) { const refs = { sourceRefName: `refs/heads/${sourceBranch}`, - targetRefName: 'refs/heads/main' // Change to 'refs/heads/master' if your repo uses master + targetRefName: 'refs/heads/master' }; const pullRequestToCreate = { @@ -183,7 +183,7 @@ Generated: ${new Date().toISOString()}` }else{ console.log("================Printing the PR details:================") console.log(` Source Branch: ${sourceBranch}`); - console.log(` Target Branch: main`); + console.log(` Target Branch: master`); console.log(` Title: Courtesy Bump of Tasks`); console.log(` Organization: ${orgUrl}`); console.log(` Project: ${project}`); @@ -407,7 +407,7 @@ async function updateUnifiedDeps(unifiedDepsPath, newUnifiedDepsPath) { * @param {Object} depsToUpdate - structure to track added/removed dependencies (formed in updateUnifiedDeps) */ async function updateTfsServerDeps() { - const repoName='AzureDevops' + const repoName='AzureDevOps' const agentTempDir=process.env.AGENT_TEMPDIRECTORY; const repoPath=path.join(agentTempDir, repoName); diff --git a/ci/get-AzDo-pat.yml b/ci/get-AzDo-pat.yml new file mode 100644 index 00000000000..f08b24562f3 --- /dev/null +++ b/ci/get-AzDo-pat.yml @@ -0,0 +1,11 @@ +steps: +- task: AzureCLI@2 + inputs: + azureSubscription: ARM - WIF - manual + scriptType: pscore + scriptLocation: inlineScript + inlineScript: | + az account set --subscription $(SUBSCRIPTION_ID) + $accessToken = az account get-access-token --resource $(RESOURCE_ID) --query accessToken --output tsv + echo "##vso[task.setvariable variable=AzDo_PAT;issecret=true]$accessToken" + displayName: Get Azure DevOps Access Token