ci: publish artifacts (#396)

* build: add version script

* build: add error message to rush

* ci: add artifact publishing

* ci: ensure doesn't run version on PR

* ci: just use version to detect whether to publish

* ci: fix variable syntax

* ci: fix job name

* build: add rimraf to lu/qnamaker

* ci: change to use build number

* ci: fix syntax

* ci: typo

* ci: syntax error

* ci: fix syntax error

* ci: fix syntax error

* ci: fix syntax error

* ci: fix syntax error...

* ci: fix syntax error

* ci: fix syntax error

* ci: fix syntax error

* ci: fallback to buildVersion

* ci: fix buildId

* ci: try a different syntax

* build: remove parallelism because deadlock

* fix: do not hang on non-tty

* revert: revert the tty fix

* build: remove doc gen in CI for time being

* build: remove docs from version

Author: christopheranderson

azure-pipelines.yml

@@ -14,52 +14,66 @@ schedules:
     include:
     - master
 
-strategy:
-  matrix:
-    linux:
-      imageName: "windows-2019"
-      nodeVersion: "12.x"
-  maxParallel: 3
+jobs:
+  - job: CLI
+    strategy:
+      matrix:
+        windows:
+          imageName: "windows-2019"
+          nodeVersion: "12.x"
+      maxParallel: 3
+    pool:
+      vmImage: $(imageName)
+    variables:
+      buildVersion: '4.7.0-preview.$(Build.BuildId)'
+      _version: ${{coalesce(variables.version, variables.buildVersion)}}
+    steps:
+    #Your build pipeline references the ‘version’ variable, which you’ve selected to be settable at queue time. Create or edit the build pipeline for this YAML file, define the variable on the Variables tab, and then select the option to make it settable at queue time. See https://go.microsoft.com/fwlink/?linkid=865971
+    - task: colinsalmcorner.colinsalmcorner-buildtasks.tag-build-task.tagBuildOrRelease@0
+      displayName: 'Tag Build with version number'
+      inputs:
+        tags: 'Version=$(_version)'
 
-pool:
-  vmImage: $(imageName)
+    - task: NodeTool@0
+      displayName: 'Use Node $(nodeVersion)'
+      inputs:
+        versionSpec: $(nodeVersion)
 
-steps:
-- powershell: 'Get-ChildItem env:* | sort-object name'
-  displayName: 'Log environment variables'
-  continueOnError: true
+    - task: Npm@1
+      displayName: 'Install rush'
+      inputs:
+        command: custom
+        verbose: false
+        customCommand: 'install --global @microsoft/rush'
 
-- task: NodeTool@0
-  displayName: 'Use Node 12.x'
-  inputs:
-    versionSpec: $(nodeVersion)
+    - script: 'rush update'
+      displayName: 'rush update'
 
-- task: Npm@1
-  displayName: 'Install rush'
-  inputs:
-    command: custom
-    verbose: false
-    customCommand: 'install --global @microsoft/rush'
+    - script: 'rush build'
+      displayName: 'rush build'
 
-- script: 'rush update'
-  displayName: 'rush update'
+    - script: 'rush test'
+      displayName: 'rush test'
 
-- script: 'rush build'
-  displayName: 'rush build'
+    - script: 'node ./common/scripts/version-and-pack.js --version $(_version)'
+      displayName: 'Version and Pack'
 
-- script: 'rush test'
-  displayName: 'rush test'
+    - task: CopyFiles@2
+      displayName: 'Copy Files to: $(Build.ArtifactStagingDirectory)'
+      inputs:
+        SourceFolder: ./.output
+        Contents: '**/*.tgz'
+        TargetFolder: '$(Build.ArtifactStagingDirectory)'
+        flattenFolders: true
 
-- task: ms.vss-governance-buildtask.governance-build-task-component-detection.ComponentGovernanceComponentDetection@0
-  displayName: 'Component Detection'
+    - task: PublishBuildArtifacts@1
+      displayName: 'Publish Artifact: drop'
+      inputs:
+        artifactName: 'drop'
 
-- script: 'rush report:coverage'
-  displayName: 'rush report:coverage'
-  enabled: false
+    - task: ms.vss-governance-buildtask.governance-build-task-component-detection.ComponentGovernanceComponentDetection@0
+      displayName: 'Component Detection'
 
-- task: PublishBuildArtifacts@1
-  enabled: false
-  displayName: 'Publish Artifact: nyc_output'
-  inputs:
-    PathtoPublish: '$(Build.SourcesDirectory)\.nyc_output'
-    ArtifactName: 'nyc_output'
+    - script: 'rush report:coverage'
+      displayName: 'rush report:coverage'
+      enabled: false

common/config/rush/command-line.json

@@ -25,6 +25,13 @@
       "summary": "Cleans non-rush content. Use rush unlink/purge/tableflip.",
       "enableParallelism": true
     },
+    {
+      "commandKind": "global",
+      "name": "pack",
+      "summary": "Create .tgz packages for publishing in CI",
+      "description":"This doesn't currently work because rush doesn't allow the passing of arbitrary commands like version numbers yet. \n Use  node ./common/scripts/version-and-pack.js --version X.X.X-tag instead.",
+      "shellCommand": "node -e \"throw new Error(`Not supported yet. \\nUse node ./common/scripts/version-and-pack.js --version X.X.X-tag instead`)\""
+    },
     /**
      * Disable parallelism because it makes the results flakier
      */

common/scripts/version-and-pack.js

@@ -0,0 +1,161 @@
+// @ts-check
+// Ensure using node 12 because of recursive mkdir
+if (
+    !process.env.GEN_CLDR_DATA_IGNORE_NODE_VERSION &&
+    process.version.split('.')[0] < 'v12'
+) {
+    console.error(`
+Your node version appears to be below v12: ${ process.version}. 
+This script will not run correctly on earlier versions of node. 
+Set 'GEN_CLDR_DATA_IGNORE_NODE_VERSION' environment variable to truthy to override`);
+}
+
+const fs = require('fs');
+const path = require('path');
+const cp = require('child_process');
+const os = require('os');
+
+// from: https://semver.org/
+const semverRegex = /^(0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*)(?:-((?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*)(?:\.(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*))*))?$/
+const npm = (process.platform === 'win32' ? 'npm.cmd' : 'npm');
+const packDirName = "./.output";
+const packDir = path.join(__dirname, "../..", packDirName);
+
+async function main() {
+    const plog = prettyLogger('version-and-pack', 'main');
+
+    // Extract version from command line
+    const version = extractVersion(process.argv);
+    if (version === undefined) {
+        throw new TypeError("You must specifiy --version as an argument");
+    }
+    if (!semverRegex.test(version)) {
+        throw new RangeError(`Version must match semver V1 format (i.e. X.X.X-label). Received: "${version}"`);
+    }
+    plog(`Setting version: ${version}`);
+
+    // Load all modules we care about from rush
+    let projects;
+    try {
+        const rushConfig = require('../../rush.json');
+        projects = rushConfig.projects;
+        plog("Loading projects:");
+        plog(JSON.stringify(projects, null, ' '));
+    } catch (e) {
+        plog("Could not load projects from rush.json");
+        throw e;
+    }
+
+    try {
+        plog("Creating output directory: " + packDir);
+        createIfNotExistSync(packDir);
+    } catch (e) {
+        plog("Could not create output directory");
+        throw e;
+    }
+    for (const project of projects) {
+        plog("Bumping version for " + project.packageName + " to " + version);
+        const pathToPackage = path.join(__dirname, "../../", project.projectFolder);
+        await exec(npm, ['version', version, '--allow-same-version'], { cwd: pathToPackage });
+
+        plog("Updating dependencies in package.json");
+        const packageJsonPath = path.join(pathToPackage, 'package.json');
+        const packageJson = JSON.parse(fs.readFileSync(packageJsonPath, 'utf8'));
+
+        for (const dep of projects) {
+            if (packageJson.dependencies[dep.packageName]) {
+                plog("Updating version in " + project.packageName + ": " + dep.packageName + " -> " + version);
+                packageJson.dependencies[dep.packageName] = version;
+            }
+        }
+
+        fs.writeFileSync(packageJsonPath, JSON.stringify(packageJson, null, ' ') + '\n', {
+            encoding: 'utf8'
+        });
+
+        plog("Packing " + project.packageName);
+        const output = await exec(npm, ['pack'], { cwd: pathToPackage });
+        const tgz = parseTgz(output);
+        plog('found tgz: ' + tgz);
+        fs.copyFileSync(path.join(pathToPackage, tgz), path.join(packDir, tgz));
+    }
+
+    plog('Complete');
+}
+
+function extractVersion(argv) {
+    for (let i = 1; i < argv.length; i++) {
+        if (argv[i - 1] === '--version') {
+            return argv[i].trim();
+        }
+    }
+}
+
+function parseTgz(text) {
+    const lines = text.split('\n');
+    for (const line of lines) {
+        if (line.endsWith('.tgz')) {
+            return line;
+        }
+    }
+}
+
+async function exec(command, args, opts) {
+    const stdout = prettyLogger(command, 'stdout');
+    const stderr = prettyLogger(command, 'stderr');
+    const error = prettyLogger(command, 'error');
+
+    return new Promise((resolve, reject) => {
+        const p = cp.spawn(command, args, opts);
+        let buffer = '';
+
+        p.stdout.on('data', data => {
+            buffer += data;
+            stdout(`[${command}][stdout]: ${data}`);
+        });
+
+        p.stderr.on('data', data => {
+            stderr(`[${command}][stderr]: ${data}`);
+        });
+
+        p.on('error', err => {
+            error(err);
+        });
+
+        p.on('close', code => {
+            if (code !== 0) {
+                return reject(new Error(`"${command} ${args.join(' ')}" returned unsuccessful error code: ${code}`));
+            } else {
+                resolve(buffer);
+            }
+        });
+    });
+}
+
+function createIfNotExistSync(path) {
+    try {
+        fs.mkdirSync(path, { recursive: true });
+    } catch (e) {
+        if (!e.code === 'EEXIST') {
+            throw e;
+        }
+    }
+}
+
+function prettyLogger(...labels) {
+    const header = `[${labels.join('][')}]: `;
+    return (content) => {
+        if (typeof content !== 'string') {
+            content = JSON.stringify(content, null, ' ');
+        }
+        const lines = content.split('\n');
+        lines.forEach((v) => console.log(header + v));
+    };
+}
+
+main().catch(err => {
+    console.error(err);
+    process.exit(1);
+}).then(() => {
+    process.exit(0);
+});

packages/cli/package.json

@@ -74,7 +74,7 @@
   "repository": "https://github.com/microsoft/botframework-cli/tree/master/packages/cli",
   "scripts": {
     "postinstall": "node scripts/postinstall.js",
-    "prepack": "npm run clean && npm run build && npm run doc:readme",
+    "prepack": "npm run clean && npm run build",
     "postpack": "rimraf oclif.manifest.json",
     "posttest": "tslint -p test -t stylish",
     "build":"tsc -b",
@@ -84,7 +84,7 @@
     "report:coverage":"npm run coverage && nyc report --reporter=text-lcov | coveralls",
     "doc": "npm run build && npm run doc:readme && rimraf oclif.manifest.json",
     "doc:readme":"oclif-dev manifest && oclif-dev readme",
-    "version": "npm run doc:readme && git add README.md"
+    "version": ""
   },
   "types": "lib/index.d.ts"
 }

packages/lu/package.json

@@ -74,6 +74,7 @@
     "mocha": "^6.2.2",
     "nock": "^11.7.0",
     "nyc": "^14.1.1",
+    "rimraf": "^2.6.3",
     "ts-node": "^8.3.0",
     "tslint": "^5.18.0",
     "typescript": "^3.5.3"

packages/qnamaker/package.json

@@ -59,6 +59,7 @@
     "readline-sync": "^1.4.9",
     "request": "^2.88.0",
     "request-promise-native": "^1.0.5",
+    "rimraf": "^2.6.3",
     "semver": "^5.5.1",
     "tslib": "^1.10.0",
     "window-size": "^1.1.0"