feat(cubejs-cli): Cube Cloud deploy implementation

paveltiunov · paveltiunov · commit b34ba53743be · 2020-06-08T23:33:41.000-07:00
diff --git a/packages/cubejs-cli/DeployDir.js b/packages/cubejs-cli/DeployDir.js
@@ -0,0 +1,53 @@
+const path = require('path');
+const fs = require('fs-extra');
+const crypto = require('crypto');
+
+class DeployDir {
+  constructor({ directory }) {
+    this.directory = directory;
+  }
+
+  filter(file) {
+    let baseName = path.basename(file);
+    return baseName !== 'node_modules' && baseName !== '.git' && baseName !== '.env';
+  }
+
+  async fileHashes(directory) {
+    directory = directory || this.directory;
+    let result = {};
+
+    const files = await fs.readdir(directory);
+    // eslint-disable-next-line no-restricted-syntax
+    for (const file of files) {
+      const filePath = path.resolve(directory, file);
+      if (!this.filter(filePath)) {
+        continue;
+      }
+      const stat = await fs.stat(filePath);
+      if (stat.isDirectory()) {
+        result = { ...result, ...await this.fileHashes(filePath) };
+      } else {
+        result[path.relative(this.directory, filePath)] = {
+          hash: await this.fileHash(filePath)
+        };
+      }
+    }
+    return result;
+  }
+
+  fileHash(file) {
+    return new Promise((resolve, reject) => {
+      const hash = crypto.createHash('sha1');
+      return fs.createReadStream(file)
+        .pipe(hash.setEncoding('hex'))
+        .on('finish', () => {
+          resolve(hash.digest('hex'))
+        })
+        .on('error', (err) => {
+          reject(err);
+        });
+    })
+  }
+}
+
+module.exports = DeployDir;
diff --git a/packages/cubejs-cli/cubejsCli.js b/packages/cubejs-cli/cubejsCli.js
@@ -13,6 +13,7 @@ const spawn = require('cross-spawn');
 const crypto = require('crypto');
 
 const templates = require('./templates');
+const { deploy } = require('./deploy');
 const { token, defaultExpiry, collect } = require('./token');
 const { requireFromPackage, event, displayError } = require('./utils');
 
@@ -257,6 +258,22 @@ program
     console.log('  $ cubejs token -e "1 day" -p foo=bar -p cool=true');
   });
 
+program
+  .command('deploy')
+  .option('-a, --auth [auth]', 'Cube Cloud Deploy Authentication Token. You can find it in Cube Cloud Deployment Settings')
+  .description('Deploy project to Cube Cloud')
+  .action(
+    (options) => deploy({ directory: process.cwd(), ...options })
+      .catch(e => displayError(e.stack || e))
+  )
+  .on('--help', () => {
+    console.log('');
+    console.log('Examples:');
+    console.log('');
+    console.log('  $ export CUBE_CLOUD_DEPLOY_AUTH=eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJkZXBsb3ltZW50SWQiOiIxIiwidXJsIjoiaHR0cHM6Ly9leGFtcGxlcy5jdWJlY2xvdWQuZGV2IiwiaWF0IjoxNTE2MjM5MDIyfQ.La3MiuqfGigfzADl1wpxZ7jlb6dY60caezgqIOoHt-c');
+    console.log('  $ cubejs deploy');
+  });
+
 if (!process.argv.slice(2).length) {
   program.help();
 }
diff --git a/packages/cubejs-cli/deploy.js b/packages/cubejs-cli/deploy.js
@@ -0,0 +1,91 @@
+const rp = require('request-promise');
+const jwt = require('jsonwebtoken');
+const fs = require('fs-extra');
+const path = require('path');
+const DeployDir = require('./DeployDir');
+const cliProgress = require('cli-progress');
+const { logStage } = require('./utils');
+
+const cloudReq = (options) => {
+  const { url, auth, ...restOptions } = options;
+  const authorization = auth || process.env.CUBE_CLOUD_DEPLOY_AUTH;
+  const payload = jwt.decode(authorization);
+  if (!payload.url || !payload.deploymentId) {
+    throw new (`Malformed token: ${authorization}`);
+  }
+  return rp({
+    headers: {
+      authorization
+    },
+    ...restOptions,
+    url: `${payload.url}/${url(payload.deploymentId)}`,
+    json: true
+  });
+};
+
+exports.deploy = async ({ directory, auth }) => {
+  const bar = new cliProgress.SingleBar({
+    format: '- Uploading files | {bar} | {percentage}% || {value} / {total} | {file}',
+    barCompleteChar: '\u2588',
+    barIncompleteChar: '\u2591',
+    hideCursor: true
+  });
+
+  const deployDir = new DeployDir({ directory });
+  const fileHashes = await deployDir.fileHashes();
+  const upstreamHashes = await cloudReq({
+    url: (deploymentId) => `build/deploy/${deploymentId}/files`,
+    method: 'GET',
+    auth
+  });
+  const { transaction, deploymentName } = await cloudReq({
+    url: (deploymentId) => `build/deploy/${deploymentId}/start-upload`,
+    method: 'POST',
+    auth
+  });
+
+  await logStage(`Deploying ${deploymentName}...`, `Cube Cloud CLI Deploy`);
+
+  const files = Object.keys(fileHashes);
+  bar.start(files.length, 0, {
+    file: ''
+  });
+
+  try {
+    for (let i = 0; i < files.length; i++) {
+      const file = files[i];
+      bar.update(i, { file });
+      if (!upstreamHashes[file] || upstreamHashes[file].hash !== fileHashes[file].hash) {
+        await cloudReq({
+          url: (deploymentId) => `build/deploy/${deploymentId}/upload-file`,
+          method: 'POST',
+          formData: {
+            transaction: JSON.stringify(transaction),
+            fileName: file,
+            file: {
+              value: fs.createReadStream(path.join(directory, file)),
+              options: {
+                filename: path.basename(file),
+                contentType: 'application/octet-stream'
+              }
+            }
+          },
+          auth
+        })
+      }
+    }
+    bar.update(files.length, { file: 'Post processing...' });
+    await cloudReq({
+      url: (deploymentId) => `build/deploy/${deploymentId}/finish-upload`,
+      method: 'POST',
+      body: {
+        transaction,
+        files: fileHashes
+      },
+      auth
+    });
+  } finally {
+    bar.stop();
+  }
+  await logStage(`Done 🎉`, `Cube Cloud CLI Deploy Success`);
+};
diff --git a/packages/cubejs-cli/package.json b/packages/cubejs-cli/package.json
@@ -31,12 +31,14 @@
   ],
   "dependencies": {
     "chalk": "^2.4.2",
+    "cli-progress": "^3.8.2",
     "commander": "^2.19.0",
     "cross-spawn": "^7.0.1",
     "fs-extra": "^8.1.0",
     "jsonwebtoken": "^8.5.1",
     "node-fetch": "^2.6.0",
-    "node-machine-id": "^1.1.10"
+    "node-machine-id": "^1.1.10",
+    "request-promise": "^4.2.5"
   },
   "devDependencies": {
     "eslint": "^6.8.0",
diff --git a/packages/cubejs-cli/utils.js b/packages/cubejs-cli/utils.js
@@ -54,3 +54,12 @@ exports.requireFromPackage = async (module) => {
   // eslint-disable-next-line global-require,import/no-dynamic-require
   return require(path.join(process.cwd(), 'node_modules', module));
 };
+
+const logStage = async (stage, eventName, props) => {
+  console.log(`- ${stage}`);
+  if (eventName) {
+    await event(eventName, props)
+  }
+};
+
+exports.logStage = logStage;
diff --git a/packages/cubejs-cli/yarn.lock b/packages/cubejs-cli/yarn.lock
@@ -649,6 +649,11 @@ bcrypt-pbkdf@^1.0.0:
   dependencies:
     tweetnacl "^0.14.3"
 
+bluebird@^3.5.0:
+  version "3.7.2"
+  resolved "https://registry.yarnpkg.com/bluebird/-/bluebird-3.7.2.tgz#9f229c15be272454ffa973ace0dbee79a1b0c36f"
+  integrity sha512-XpNj6GDQzdfW+r2Wnn7xiSAd7TM3jzkxGXBGTtWKuSXv1xUV+azxAm8jdWZN06QTQk+2N2XB9jRDkvbmQmcRtg==
+
 brace-expansion@^1.1.7:
   version "1.1.11"
   resolved "https://registry.yarnpkg.com/brace-expansion/-/brace-expansion-1.1.11.tgz#3c7fcbf529d87226f3d2f52b966ff5271eb441dd"
@@ -793,6 +798,14 @@ cli-cursor@^3.1.0:
   dependencies:
     restore-cursor "^3.1.0"
 
+cli-progress@^3.8.2:
+  version "3.8.2"
+  resolved "https://registry.yarnpkg.com/cli-progress/-/cli-progress-3.8.2.tgz#abaf1fc6d6401351f16f068117a410554a0eb8c7"
+  integrity sha512-qRwBxLldMSfxB+YGFgNRaj5vyyHe1yMpVeDL79c+7puGujdKJHQHydgqXDcrkvQgJ5U/d3lpf6vffSoVVUftVQ==
+  dependencies:
+    colors "^1.1.2"
+    string-width "^4.2.0"
+
 cli-width@^2.0.0:
   version "2.2.0"
   resolved "https://registry.yarnpkg.com/cli-width/-/cli-width-2.2.0.tgz#ff19ede8a9a5e579324147b0c11f0fbcbabed639"
@@ -849,6 +862,11 @@ color-name@~1.1.4:
   resolved "https://registry.yarnpkg.com/color-name/-/color-name-1.1.4.tgz#c2a09a87acbde69543de6f63fa3995c826c536a2"
   integrity sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==
 
+colors@^1.1.2:
+  version "1.4.0"
+  resolved "https://registry.yarnpkg.com/colors/-/colors-1.4.0.tgz#c50491479d4c1bdaed2c9ced32cf7c7dc2360f78"
+  integrity sha512-a+UqTh4kgZg/SlGvfbzDHpgRu7AAQOmmqRHJnxhRZICKFUT91brVhNNt58CMWU9PsBbv3PDCZUHbVxuDiH2mtA==
+
 combined-stream@^1.0.6, combined-stream@~1.0.6:
   version "1.0.7"
   resolved "https://registry.yarnpkg.com/combined-stream/-/combined-stream-1.0.7.tgz#2d1d24317afb8abe95d6d2c0b07b57813539d828"
@@ -3724,6 +3742,13 @@ request-promise-core@1.1.2:
   dependencies:
     lodash "^4.17.11"
 
+request-promise-core@1.1.3:
+  version "1.1.3"
+  resolved "https://registry.yarnpkg.com/request-promise-core/-/request-promise-core-1.1.3.tgz#e9a3c081b51380dfea677336061fea879a829ee9"
+  integrity sha512-QIs2+ArIGQVp5ZYbWD5ZLCY29D5CfWizP8eWnm8FoGD1TX61veauETVQbrV60662V0oFBkrDOuaBI8XgtuyYAQ==
+  dependencies:
+    lodash "^4.17.15"
+
 request-promise-native@^1.0.5:
   version "1.0.7"
   resolved "https://registry.yarnpkg.com/request-promise-native/-/request-promise-native-1.0.7.tgz#a49868a624bdea5069f1251d0a836e0d89aa2c59"
@@ -3733,6 +3758,16 @@ request-promise-native@^1.0.5:
     stealthy-require "^1.1.1"
     tough-cookie "^2.3.3"
 
+request-promise@^4.2.5:
+  version "4.2.5"
+  resolved "https://registry.yarnpkg.com/request-promise/-/request-promise-4.2.5.tgz#186222c59ae512f3497dfe4d75a9c8461bd0053c"
+  integrity sha512-ZgnepCykFdmpq86fKGwqntyTiUrHycALuGggpyCZwMvGaZWgxW6yagT0FHkgo5LzYvOaCNvxYwWYIjevSH1EDg==
+  dependencies:
+    bluebird "^3.5.0"
+    request-promise-core "1.1.3"
+    stealthy-require "^1.1.1"
+    tough-cookie "^2.3.3"
+
 request@^2.87.0:
   version "2.88.0"
   resolved "https://registry.yarnpkg.com/request/-/request-2.88.0.tgz#9c2fca4f7d35b592efe57c7f0a55e81052124fef"
@@ -4205,7 +4240,7 @@ string-width@^3.0.0:
     is-fullwidth-code-point "^2.0.0"
     strip-ansi "^5.0.0"
 
-string-width@^4.1.0:
+string-width@^4.1.0, string-width@^4.2.0:
   version "4.2.0"
   resolved "https://registry.yarnpkg.com/string-width/-/string-width-4.2.0.tgz#952182c46cc7b2c313d1596e623992bd163b72b5"
   integrity sha512-zUz5JD+tgqtuDjMhwIg5uFVV3dtqZ9yQJlZVfq4I01/K5Paj5UHj7VyrQOJvzawSVlKpObApbfD0Ed6yJc+1eg==
