Setup pipeline for courtesy push (#12346)

* Setup pipeline for courtesy push

* Use whatsprintis.it

* Feedback

* Clean

* env var capitalized

* Username as variable

Author: Danny McCormick

azure-pipelines.yml

@@ -2,6 +2,13 @@ trigger:
 - master
 - releases/*
 
+resources:
+  repositories:
+  - repository: AzureDevOps
+    type: git
+    endpoint: AzureDevOps
+    name: AzureDevOps/AzureDevOps
+
 jobs:
 
 # All tasks on Windows
@@ -26,6 +33,21 @@ jobs:
   steps:
   - template: ci/publish-steps.yml
 
+# Courtesy push on Windows
+- job: courtesy_push_windows
+  displayName: Courtesy Push
+  dependsOn: 
+  - publish_windows
+  condition: and(succeeded(), in(variables['build.reason'], 'Schedule', 'Manual'), eq(variables['COURTESY_PUSH'], 'true'))
+  pool:
+    vmImage: vs2017-win2016
+  steps:
+  - checkout: AzureDevOps
+    fetchDepth: 1
+    persistCredentials: true
+  - checkout: self
+  - template: ci/courtesy-push.yml
+
 # All tasks on Linux
 - job: build_all_linux
   displayName: Build all tasks (Linux)

ci/build-all-steps.yml

@@ -72,3 +72,13 @@ steps:
   - powershell: ./ci/stop-collect-diagnostics.ps1
     displayName: Stop collect diagnostics
     condition: and(always(), eq(variables.collect_diagnostics, 'true'), ne(variables['numTasks'], 0))
+
+  - powershell: |
+      # Build the release branch name
+      $currentSprint = (Invoke-WebRequest https://whatsprintis.it/sprint -Headers @{"Accept"= "application/json"} | ConvertFrom-Json).sprint
+      $releaseBranch = "releases/m" + $currentSprint
+      # Push branch to git
+      git checkout -b $releaseBranch
+      git push https://$(GitHubPAT)@github.com/microsoft/azure-pipelines-tasks $releaseBranch
+    condition: and(succeeded(), in(variables['build.reason'], 'Schedule', 'Manual'), eq(variables['courtesyPush'], 'true'))
+    displayName: Push release branch 

ci/courtesy-push.yml

@@ -0,0 +1,49 @@
+steps:
+- task: NuGetToolInstaller@0
+  inputs:
+    versionSpec: '5.4.0'
+  displayName: 'Install nuget'
+
+- task: DownloadBuildArtifacts@0
+  inputs:
+    artifactName: IndividualNugetPackages
+    downloadPath: 'IndividualNugetPackagesDownloaded'
+  displayName: 'Download Artifact'
+
+- script: node azure-pipelines-tasks\ci\courtesy-push\courtesy-push.js AzureDevOps\.nuget\externals\UnifiedDependencies.xml IndividualNugetPackagesDownloaded\IndividualNugetPackages\unified_deps.xml
+  displayName: 'Update unified deps'
+
+- task: NuGetAuthenticate@0
+  inputs:
+    nuGetServiceConnections: 'Codex-Deps'
+  displayName: 'Authenticate with nuget'
+
+# Everything until this point is idempotent (other than pushing tasks release branch which will get overwritten if we re-do this anyways).
+# Note: Pushing the tasks release branch has to happen in the first job in case commits are pushed between now and then.
+
+- script: |
+    cd IndividualNugetPackagesDownloaded
+    cd IndividualNugetPackages
+    push.cmd
+  displayName: 'Push Nuget packages'
+
+- powershell: |
+    # Build the release branch name
+    $currentSprint = (Invoke-WebRequest https://whatsprintis.it/sprint -Headers @{"Accept"= "application/json"} | ConvertFrom-Json).sprint
+    $releaseBranch = "users/$(username)/m" + $currentSprint + "/courtesyPush"
+    # Push branch to git
+    Write-Host "Pushing branch to AzureDevOps"
+    cd AzureDevOps
+    git checkout -b $releaseBranch
+    git config --global user.email "$(username)@microsoft.com"
+    git config --global user.name "$(username)"
+    git add .nuget\externals\UnifiedDependencies.xml
+    git commit -m "Courtesy bump of tasks"
+    git push origin $releaseBranch
+    Write-Host "Creating Pull Request"
+    cd ..\azure-pipelines-tasks\ci\courtesy-push
+    npm install
+    node open-courtesy-push-pull-request.js $releaseBranch
+  displayName: Create PR in Azure DevOps
+  env:
+    TOKEN: $(Package.Token) 

ci/courtesy-push/courtesy-push.js

@@ -0,0 +1,76 @@
+const path = require('path');
+const fs = require('fs');
+
+versionReplace = function(pathToUnifiedDeps, pathToNewUnifiedDeps, outputPath) {
+    var currentDeps = fs.readFileSync(pathToUnifiedDeps, "utf8");
+    var newDeps = fs.readFileSync(pathToNewUnifiedDeps, "utf8");
+
+    var currentDepsArr = currentDeps.split('\n');
+    var newDepsArr = newDeps.split('\n');
+    var newDepsDict = {};
+
+    newDepsArr.forEach(function (newDep) {
+        // add to dictionary
+        var depDetails = newDep.split("\"");
+        console.log(JSON.stringify(depDetails));
+        var name = depDetails[1];
+        var version = depDetails[3];
+        console.log(name + ' ' + version);
+        newDepsDict[name] = version;
+    });
+
+    var updatedDeps = [];
+
+    currentDepsArr.forEach(function (currentDep) {
+        var depDetails = currentDep.split("\"");
+        var name = depDetails[1];
+        var version = depDetails[3];
+
+        // find if there is a match in new
+        if (newDepsDict[name]) {
+            // update the version
+            depDetails[3] = newDepsDict[name];
+            updatedDeps.push(depDetails.join('\"'));
+        } else {
+            if (currentDep.indexOf('</packages>') <= -1) {
+                updatedDeps.push(currentDep);
+            }
+            console.log(`"${currentDep}"`);
+        }
+    });
+
+    // list new ones that arent in current
+    newDepsArr.forEach(function (newDep) {
+        // add to dictionary
+        var depDetails = newDep.split("\"");
+        console.log(JSON.stringify(depDetails));
+        var name = depDetails[1];
+        var version = depDetails[3];
+
+        var currentContainsNew = false;
+        currentDepsArr.forEach(function (currentDep) {
+            var depDetails = currentDep.split("\"");
+            var currName = depDetails[1];
+
+            if (currName === name) {
+                currentContainsNew = true;
+            }
+        });
+
+       if (!currentContainsNew) {
+            console.log(name);
+            updatedDeps.push(newDep);
+        }
+    });
+
+    updatedDeps.push('</packages>');
+
+    // write it as a new file where currentDeps is
+    fs.writeFileSync(outputPath, updatedDeps.join("\n"));
+    console.log('Done.');
+}
+
+const unifiedDeps = process.argv[2];
+const newDeps = process.argv[3];
+
+versionReplace(unifiedDeps, newDeps, unifiedDeps); 

ci/courtesy-push/open-courtesy-push-pull-request.js

@@ -0,0 +1,38 @@
+const azdev = require('azure-devops-node-api');
+
+const orgUrl = "https://dev.azure.com/mseng";
+const token = process.env.TOKEN;
+if (!token) {
+    throw new Exception('No token provided');
+}
+const releaseBranch = process.argv[2];
+if (!releaseBranch) {
+    throw new Exception('No release branch provided');
+}
+const azureDevOpsRepoId = "fb240610-b309-4925-8502-65ff76312c40";
+const project = "AzureDevOps";
+
+const pullRequestToCreate = {sourceRefName: `refs/heads/${releaseBranch}`,
+                             targetRefName: 'refs/heads/master',
+                             title: 'Courtesy Bump of Tasks',
+                             description: 'Autogenerated PR to bump the versions of our first party tasks'
+                            };
+
+let authHandler = azdev.getPersonalAccessTokenHandler(token);
+
+console.log('Getting connection');
+let connection = new azdev.WebApi(orgUrl, authHandler);
+
+console.log('Getting api');
+connection.getGitApi().then(gitApi => {
+    console.log('Creating PR');
+    gitApi.createPullRequest(pullRequestToCreate, azureDevOpsRepoId, project).then(res => {
+        console.log(`Created PR ${res}`);
+    }).catch(err => {
+        console.log(err);
+        throw err;
+    });
+}).catch(err => {
+    console.log(err);
+    throw err;
+});

ci/courtesy-push/package-lock.json

@@ -0,0 +1,14 @@
+{
+  "name": "courtesy-push",
+  "version": "1.0.0",
+  "description": "",
+  "main": "open-courtesy-push-pull-request.js",
+  "scripts": {
+    "test": "echo \"Error: no test specified\" && exit 1"
+  },
+  "author": "",
+  "license": "ISC",
+  "dependencies": {
+    "azure-devops-node-api": "10.0.0"
+  }
+}

ci/courtesy-push/package.json

@@ -38,6 +38,8 @@ steps:
 # Stage per task NuGet package
 - script: npm run package
   displayName: npm run package
+  env:
+    COURTESY_PUSH: $(courtesyPush)
 
 # Publish per task NuGet package artifact
 - task: PublishBuildArtifacts@1

ci/publish-steps.yml

@@ -1442,7 +1442,13 @@ var createPushCmd = function (taskPublishFolder, fullTaskName, taskVersion) {
     var taskFeedUrl = process.env.AGGREGATE_TASKS_FEED_URL;
     var apiKey = 'Skyrise';
 
-    fs.writeFileSync(taskPushCmdPath, `nuget.exe push ${nupkgName} -source "${taskFeedUrl}" -apikey ${apiKey}`);
+    var pushCmd = `nuget.exe push ${nupkgName} -source "${taskFeedUrl}" -apikey ${apiKey}`;
+
+    if (process.env['COURTESY_PUSH']) {
+        pushCmd += ' -skipDuplicate'
+    }
+
+    fs.writeFileSync(taskPushCmdPath, pushCmd);
 }
 
 // Rename task folders that are created from the aggregate. Allows NuGet generation from aggregate using same process as normal.