Merge branch 'master' into fix-received-misspelled

Author: MateusFrFreitas

docs/pages/reference/data-model/pre-aggregations.mdx

@@ -926,6 +926,12 @@ To have a pre-aggregation rebuild at a specific time of day, you can use a CRON
 
 You can also use `every` with `sql`:
 
+<WarningBox>
+
+Using CRON strings and `sql` is not supported and will result in a compilation error.
+
+</WarningBox>
+
 <CodeTabs>
 
 ```javascript
@@ -1756,4 +1762,4 @@ cubes:
 [ref-aggregating-indexes]: /product/caching/using-pre-aggregations#aggregating-indexes
 [ref-pre-aggs]: /product/caching/using-pre-aggregations
 [ref-ref-cubes]: /reference/data-model/cube
-[ref-custom-granularity]: /reference/data-model/dimensions#granularities
+[ref-custom-granularity]: /reference/data-model/dimensions#granularities

packages/cubejs-backend-cloud/src/cloud.ts

@@ -2,6 +2,7 @@ import fetch, { RequestInit } from 'node-fetch';
 import FormData from 'form-data';
 import path from 'path';
 import { ReadStream } from 'fs';
+import { DotenvParseOutput } from '@cubejs-backend/dotenv';
 
 export type AuthObject = {
   auth: string,
@@ -113,22 +114,18 @@ export class CubeCloudClient {
     const formData = new FormData();
     formData.append('transaction', JSON.stringify(transaction));
     formData.append('fileName', fileName);
-    formData.append('file', {
-      value: data,
-      options: {
-        filename: path.basename(fileName),
-        contentType: 'application/octet-stream'
-      }
+    formData.append('file', data, {
+      filename: path.basename(fileName),
+      contentType: 'application/octet-stream'
     });
 
     // Get the form data buffer and headers
-    const formDataBuffer = formData.getBuffer();
     const formDataHeaders = formData.getHeaders();
 
     return this.request({
       url: (deploymentId: string) => `build/deploy/${deploymentId}/upload-file${this.extendRequestByLivePreview()}`,
       method: 'POST',
-      body: formDataBuffer,
+      body: formData,
       headers: {
         ...formDataHeaders,
       },
@@ -149,9 +146,10 @@ export class CubeCloudClient {
     });
   }
 
-  public setEnvVars({ envVariables, auth }: { envVariables: any, auth?: AuthObject }) {
+  public setEnvVars({ envVariables, auth, replaceEnv }: { envVariables: DotenvParseOutput, auth?: AuthObject, replaceEnv?: boolean }) {
+    const params = new URLSearchParams({ replaceEnv: Boolean(replaceEnv).toString() });
     return this.request({
-      url: (deploymentId) => `build/deploy/${deploymentId}/set-env`,
+      url: (deploymentId) => `build/deploy/${deploymentId}/set-env?${params.toString()}`,
       method: 'POST',
       body: JSON.stringify({ envVariables: JSON.stringify(envVariables) }),
       headers: {

packages/cubejs-backend-cloud/src/deploy.ts

@@ -1,6 +1,7 @@
 import crypto from 'crypto';
 import path from 'path';
 import fs from 'fs-extra';
+import { DotenvParseOutput } from '@cubejs-backend/dotenv';
 import { CubeCloudClient } from './cloud';
 
 type DeployDirectoryOptions = {
@@ -64,10 +65,10 @@ export class DeployDirectory {
 }
 
 type DeployHooks = {
-  onStart?: Function,
-  onUpdate?: Function,
-  onUpload?: Function,
-  onFinally?: Function
+  onStart?: (deploymentName: string, files: string[]) => void,
+  onUpdate?: (i: number, { file }: { file: string}) => void,
+  onUpload?: (files: string[], file: string) => void,
+  onFinally?: () => void
 };
 
 export interface DeployResponse {
@@ -78,6 +79,7 @@ export interface DeployResponse {
 export class DeployController {
   public constructor(
     protected readonly cubeCloudClient: CubeCloudClient,
+    protected readonly envs: { envVariables?: DotenvParseOutput, replaceEnv?: boolean } = {},
     protected readonly hooks: DeployHooks = {}
   ) {
   }
@@ -90,6 +92,13 @@ export class DeployController {
     const upstreamHashes = await this.cubeCloudClient.getUpstreamHashes();
     const { transaction, deploymentName } = await this.cubeCloudClient.startUpload();
 
+    if (this.envs.envVariables) {
+      const { envVariables, replaceEnv } = this.envs;
+      if (Object.keys(this.envs.envVariables).length) {
+        await this.cubeCloudClient.setEnvVars({ envVariables, replaceEnv });
+      }
+    }
+
     const files = Object.keys(fileHashes);
     const fileHashesPosix: Record<string, any> = {};
     if (this.hooks.onStart) this.hooks.onStart(deploymentName, files);

packages/cubejs-base-driver/src/BaseDriver.ts

@@ -470,20 +470,20 @@ export abstract class BaseDriver implements DriverInterface {
     }
   }
 
-  public getSchemas() {
+  public getSchemas(): Promise<QuerySchemasResult[]> {
     const query = this.getSchemasQuery();
     return this.query<QuerySchemasResult>(query);
   }
 
-  public getTablesForSpecificSchemas(schemas: QuerySchemasResult[]) {
+  public getTablesForSpecificSchemas(schemas: QuerySchemasResult[]): Promise<QueryTablesResult[]> {
     const schemasPlaceholders = schemas.map((_, idx) => this.param(idx)).join(', ');
     const schemaNames = schemas.map(s => s.schema_name);
 
     const query = this.getTablesForSpecificSchemasQuery(schemasPlaceholders);
     return this.query<QueryTablesResult>(query, schemaNames);
   }
 
-  public async getColumnsForSpecificTables(tables: QueryTablesResult[]) {
+  public async getColumnsForSpecificTables(tables: QueryTablesResult[]): Promise<QueryColumnsResult[]> {
     const groupedBySchema: Record<string, string[]> = {};
     tables.forEach((t) => {
       if (!groupedBySchema[t.schema_name]) {

packages/cubejs-bigquery-driver/src/BigQueryDriver.ts

@@ -222,14 +222,14 @@ export class BigQueryDriver extends BaseDriver implements DriverInterface {
     return dataSetsColumns.reduce((prev, current) => Object.assign(prev, current), {});
   }
 
-  public async getSchemas(): Promise<QuerySchemasResult[]> {
+  public override async getSchemas(): Promise<QuerySchemasResult[]> {
     const dataSets = await this.bigquery.getDatasets();
     return dataSets[0].filter((dataSet) => dataSet.id).map((dataSet) => ({
       schema_name: dataSet.id!,
     }));
   }
 
-  public async getTablesForSpecificSchemas(schemas: QuerySchemasResult[]): Promise<QueryTablesResult[]> {
+  public override async getTablesForSpecificSchemas(schemas: QuerySchemasResult[]): Promise<QueryTablesResult[]> {
     try {
       const allTablePromises = schemas.map(async schema => {
         const tables = await this.getTablesQuery(schema.schema_name);
@@ -247,7 +247,7 @@ export class BigQueryDriver extends BaseDriver implements DriverInterface {
     }
   }
 
-  public async getColumnsForSpecificTables(tables: QueryTablesResult[]): Promise<QueryColumnsResult[]> {
+  public override async getColumnsForSpecificTables(tables: QueryTablesResult[]): Promise<QueryColumnsResult[]> {
     try {
       const allColumnPromises = tables.map(async table => {
         const tableName = `${table.schema_name}.${table.table_name}`;

packages/cubejs-cli/package.json

@@ -31,6 +31,7 @@
   ],
   "dependencies": {
     "@cubejs-backend/dotenv": "^9.0.2",
+    "@cubejs-backend/cloud": "1.1.13",
     "@cubejs-backend/schema-compiler": "1.1.15",
     "@cubejs-backend/shared": "1.1.12",
     "chalk": "^2.4.2",

packages/cubejs-cli/src/command/auth.ts

@@ -1,7 +1,7 @@
 import type { CommanderStatic } from 'commander';
+import { Config } from '@cubejs-backend/cloud';
 
 import { displayError, event } from '../utils';
-import { Config } from '../config';
 
 const authenticate = async (currentToken: string) => {
   const config = new Config();

packages/cubejs-cli/src/command/deploy.ts

@@ -1,138 +1,72 @@
-import FormData from 'form-data';
 import fs from 'fs-extra';
 import path from 'path';
 import cliProgress from 'cli-progress';
 import { CommanderStatic } from 'commander';
+import { AuthObject, CubeCloudClient, DeployController } from '@cubejs-backend/cloud';
+import { ConfigCli } from '../config';
 
-import { DeployDirectory } from '../deploy';
 import { logStage, displayError, event } from '../utils';
-import { Config } from '../config';
 
-interface Hashes {
-  [key: string]: {
-    hash: string;
-  };
-}
-
-interface CloudReqResult {
-  transaction: string;
-  deploymentName: string;
-}
+type DeployOptions = {
+  directory: string,
+  auth: AuthObject,
+  uploadEnv: boolean,
+  replaceEnv: boolean,
+  token: string
+};
 
-const deploy = async ({ directory, auth, uploadEnv, token }: any) => {
+const deploy = async ({ directory, auth, uploadEnv, replaceEnv, token }: DeployOptions) => {
   if (!(await fs.pathExists(path.join(process.cwd(), 'node_modules', '@cubejs-backend/server-core')))) {
     await displayError(
       '@cubejs-backend/server-core dependency not found. Please run deploy command from project root directory and ensure npm install has been run.'
     );
   }
 
+  const config = new ConfigCli();
   if (token) {
-    const config = new Config();
     await config.addAuthToken(token);
-
-    await event({
-      event: 'Cube Cloud CLI Authenticate'
-    });
-
+    await event({ event: 'Cube Cloud CLI Authenticate' });
     console.log('Token successfully added!');
   }
 
-  const config = new Config();
   const bar = new cliProgress.SingleBar({
     format: '- Uploading files | {bar} | {percentage}% || {value} / {total} | {file}',
     barCompleteChar: '\u2588',
     barIncompleteChar: '\u2591',
     hideCursor: true
   });
 
-  const deployDir = new DeployDirectory({ directory });
-  const fileHashes: any = await deployDir.fileHashes();
-
-  const upstreamHashes: Hashes = await config.cloudReq({
-    url: (deploymentId: string) => `build/deploy/${deploymentId}/files`,
-    method: 'GET',
-    auth
-  });
-
-  const { transaction, deploymentName }: CloudReqResult = await config.cloudReq({
-    url: (deploymentId: string) => `build/deploy/${deploymentId}/start-upload`,
-    method: 'POST',
-    auth
-  });
-
-  if (uploadEnv) {
-    const envVariables = await config.envFile(`${directory}/.env`);
-    await config.cloudReq({
-      url: (deploymentId) => `build/deploy/${deploymentId}/set-env`,
-      method: 'POST',
-      body: JSON.stringify({
-        envVariables: JSON.stringify(envVariables),
-      }),
-      headers: {
-        'Content-type': 'application/json'
-      },
-      auth
-    });
-  }
-
-  await logStage(`Deploying ${deploymentName}...`, 'Cube Cloud CLI Deploy');
-
-  const files = Object.keys(fileHashes);
-  const fileHashesPosix = {};
-
-  bar.start(files.length, 0, {
-    file: ''
-  });
-
-  try {
-    for (let i = 0; i < files.length; i++) {
-      const file = files[i];
+  const envVariables = uploadEnv || replaceEnv ? await config.envFile(`${directory}/.env`) : {};
+
+  const cubeCloudClient = new CubeCloudClient(auth || (await config.deployAuthForCurrentDir()));
+  const deployController = new DeployController(cubeCloudClient, { envVariables, replaceEnv }, {
+    onStart: async (deploymentName, files) => {
+      await logStage(`Deploying ${deploymentName}...`, 'Cube Cloud CLI Deploy');
+      bar.start(files.length, 0, {
+        file: ''
+      });
+    },
+    onUpdate: (i, { file }) => {
       bar.update(i, { file });
-
-      const filePosix = file.split(path.sep).join(path.posix.sep);
-      fileHashesPosix[filePosix] = fileHashes[file];
-
-      if (!upstreamHashes[filePosix] || upstreamHashes[filePosix].hash !== fileHashes[file].hash) {
-        const formData = new FormData();
-        formData.append('transaction', JSON.stringify(transaction));
-        formData.append('fileName', filePosix);
-        formData.append('file', fs.createReadStream(path.join(directory, file)), {
-          filename: path.basename(file),
-          contentType: 'application/octet-stream'
-        });
-
-        await config.cloudReq({
-          url: (deploymentId: string) => `build/deploy/${deploymentId}/upload-file`,
-          method: 'POST',
-          body: formData,
-          auth,
-          headers: formData.getHeaders()
-        });
-      }
+    },
+    onUpload: (files) => {
+      bar.update(files.length, { file: 'Post processing...' });
+    },
+    onFinally: () => {
+      bar.stop();
     }
-    bar.update(files.length, { file: 'Post processing...' });
-    await config.cloudReq({
-      url: (deploymentId: string) => `build/deploy/${deploymentId}/finish-upload`,
-      method: 'POST',
-      body: JSON.stringify({
-        transaction,
-        files: fileHashesPosix
-      }),
-      headers: { 'Content-type': 'application/json' },
-      auth
-    });
-  } finally {
-    bar.stop();
-  }
+  });
 
+  await deployController.deploy(directory);
   await logStage('Done 🎉', 'Cube Cloud CLI Deploy Success');
 };
 
 export function configureDeployCommand(program: CommanderStatic) {
   program
     .command('deploy')
     .description('Deploy project to Cube Cloud')
-    .option('--upload-env', 'Upload .env file to CubeCloud')
+    .option('--upload-env', 'Use .env file to populate environment variables in Cube Cloud. Only set them once during the very first upload for this deployment')
+    .option('--replace-env', 'Use .env file to populate environment variables in Cube Cloud. Replace them with new ones during every upload for this deployment')
     .option('--token <token>', 'Add auth token to CubeCloud')
     .option('--directory [path]', 'Specify path to conf directory', './')
     .action(