Merge branch 'master' into dremio-date-interval-fix

Author: wardsi

docs/pages/product/deployment/cloud/byoc/_meta.js

@@ -1,3 +1,4 @@
 module.exports = {
   "aws": "AWS",
+  "azure": "Azure",
 }

docs/pages/product/deployment/cloud/byoc/azure.mdx

@@ -0,0 +1,88 @@
+# Deploying Cube Cloud BYOC on Azure
+
+With Bring Your Own Cloud (BYOC) on Azure, all the components interacting with private data are deployed on
+the customer infrastructure on Azure and managed by the Cube Cloud Control Plane via the Cube Cloud Operator.
+This document provides step-by-step instructions for deploying Cube Cloud BYOC on Azure.
+
+## Overall Design
+Cube Cloud will gain access to your Azure account via the Cube Cloud Provisioner Enterprise App.
+
+It will leverage a dedicated subscription where it will create a new Resource
+Group and bootstrap all the necessary infrastructure. At the center of the BYOC
+infrastructure are two AKS clusters that provide compute resources for the Cube
+Store and all Cube deployments you configure in the Cube Cloud UI. These AKS
+clusters will have a Cube Cloud operator installed in them that is connected to
+the Cube Cloud Control Plane. The Cube Cloud Operator receives instructions
+from the Control Plane and dynamically creates or destroys all the necessary
+Kubernetes resources required to support your Cube deployments.
+
+<div style={{ textAlign: "center" }}>
+  <img
+    alt="High-level diagram of Cube Cloud resources deployed on Azure"
+    src="https://ucarecdn.com/6d0f12db-086c-4274-b165-da68ccc381a9/"
+    style={{ border: "none" }}
+    width="100%"
+  />
+</div>
+
+## Prerequisites
+
+The bulk of provisioning work will be done remotely by Cube Cloud automation.
+However, to get started, you'll need to provide Cube with the necessary access
+along with some additional information that includes:
+
+- **Azure Tenant ID** - the Entra ID of your Azure account
+- **Azure Subscription ID** - The target subscription where Cube Cloud will be granted admin permissions to provision the BYOC infrastructure
+- **Region** - The target Azure region where Cube Cloud BYOC will be installed
+
+## Provisioning access
+
+### Add Cube tenant to your organization
+
+First you should add the Cube Cloud tenant to your organization. To do this,
+open the [Azure Portal][azure-console] and go to&nbsp;<Btn>Azure Active
+Directory</Btn> →&nbsp;<Btn>External Identities</Btn> →&nbsp;<Btn>Cross-tenant
+access settings</Btn> →&nbsp;<Btn>Organizational Settings</Btn>
+→&nbsp;<Btn>Add Organization</Btn>.
+
+For Tenant ID, enter `197e5263-87f4-4ce1-96c4-351b0c0c714a`.
+
+Make sure that&nbsp;<Btn>B2B Collaboration</Btn> →&nbsp;<Btn>Inbound Access</Btn>
+→&nbsp;<Btn>Applications</Btn> is set to&nbsp;<Btn>Allows access</Btn>.
+
+### Register Cube Cloud service principal at your organization
+
+To register the Cube Cloud service principal for your organization, follow these
+steps:
+
+1.  Log in with an account that has permissions to register Enterprise
+    applications.
+2.  Open a browser tab and go to the following URL, replacing `<TENANT_ID>` with
+    your tenant ID:
+    `https://login.microsoftonline.com/<TENANT_ID>/oauth2/authorize?client_id=0c5d0d4b-6cee-402e-9a08-e5b79f199481&response_type=code&redirect_uri=https%3A%2F%2Fwww.microsoft.com%2F`
+3.  The Cube Cloud service principal has specific credentials. Check that the
+    following details match exactly what you see on the dialog box that pops up:
+
+- Client ID: `d1c59948-4d4a-43dc-8d04-c0df8795ae19`
+- Name: `cube-cloud-byoc-provisioner`
+
+Once you have confirmed that all the information is correct,
+select&nbsp;<Btn>Consent on behalf of your organization</Btn> and
+click&nbsp;<Btn>Accept</Btn>.
+
+### Grant admin permissions on your BYOC Azure Subscription to the cube-cloud-byoc-provisioner
+
+On the [Azure Portal][azure-console], go to&nbsp;<Btn>Subscriptions</Btn>
+→ _Your BYOC Subscription_ →&nbsp;<Btn>IAM</Btn>→&nbsp;<Btn>Role Assignment</Btn>
+ and assing `Contributor` and `Role Based Access Control Administrator` to the `cube-cloud-byoc-provisioner`
+ Service Principal.
+
+<Screenshot src="https://ucarecdn.com/e1e917cd-6992-4864-b20e-0fbf7688a7e5/"/>
+
+## Deployment
+
+The actual deployment will be done by Cube Cloud automation. All that's left to
+do is notify your Cube contact point that access has been granted, and pass
+along your Azure Tenant/Subscription/Region information.
+
+[azure-console]: https://portal.azure.com

docs/pages/product/deployment/cloud/continuous-deployment.mdx

@@ -34,6 +34,13 @@ First, ensure your deployment is configured to deploy with Git. Then connect
 your GitHub repository to your deployment by clicking the <Btn>Connect to
 GitHub</Btn> button, and selecting your repository.
 
+<WarningBox>
+
+If your organization uses SAML SSO for GitHub authentication, make sure to start an active SAML 
+session prior to connecting to your GitHub account from Cube.
+
+<WarningBox>
+
 <div style={{ textAlign: "center" }}>
   <img
     src="https://ucarecdn.com/3dab6d90-6bd9-44a5-ae4d-d5579a08dd89/"

packages/cubejs-api-gateway/openspec.yml

@@ -170,6 +170,20 @@ components:
           type: array
           items:
             type: "string"
+    V1CubeMetaHierarchy:
+      type: "object"
+      required:
+        - name
+        - levels
+      properties:
+        name:
+          type: "string"
+        title:
+          type: "string"
+        levels:
+          type: "array"
+          items:
+            type: "string"
     V1CubeMeta:
       type: "object"
       required:
@@ -209,6 +223,10 @@ components:
           type: "array"
           items:
             $ref: "#/components/schemas/V1CubeMetaFolder"
+        hierarchies:
+          type: "array"
+          items:
+            $ref: "#/components/schemas/V1CubeMetaHierarchy"
     V1CubeMetaType:
       type: "string"
       description: Type of cube

packages/cubejs-schema-compiler/src/compiler/DataSchemaCompiler.js

@@ -23,7 +23,9 @@ export class DataSchemaCompiler {
     this.cubeCompilers = options.cubeCompilers || [];
     this.contextCompilers = options.contextCompilers || [];
     this.transpilers = options.transpilers || [];
+    this.viewCompilers = options.viewCompilers || [];
     this.preTranspileCubeCompilers = options.preTranspileCubeCompilers || [];
+    this.viewCompilationGate = options.viewCompilationGate;
     this.cubeNameCompilers = options.cubeNameCompilers || [];
     this.extensions = options.extensions || {};
     this.cubeFactory = options.cubeFactory;
@@ -93,7 +95,10 @@ export class DataSchemaCompiler {
     const compilePhase = (compilers) => this.compileCubeFiles(compilers, transpile(), errorsReport);
 
     return compilePhase({ cubeCompilers: this.cubeNameCompilers })
-      .then(() => compilePhase({ cubeCompilers: this.preTranspileCubeCompilers }))
+      .then(() => compilePhase({ cubeCompilers: this.preTranspileCubeCompilers.concat([this.viewCompilationGate]) }))
+      .then(() => (this.viewCompilationGate.shouldCompileViews() ?
+        compilePhase({ cubeCompilers: this.viewCompilers })
+        : Promise.resolve()))
       .then(() => compilePhase({
         cubeCompilers: this.cubeCompilers,
         contextCompilers: this.contextCompilers,

packages/cubejs-schema-compiler/src/compiler/PrepareCompiler.ts

@@ -20,6 +20,7 @@ import { JoinGraph } from './JoinGraph';
 import { CubeToMetaTransformer } from './CubeToMetaTransformer';
 import { CompilerCache } from './CompilerCache';
 import { YamlCompiler } from './YamlCompiler';
+import { ViewCompilationGate } from './ViewCompilationGate';
 
 export type PrepareCompilerOptions = {
   nativeInstance?: NativeInstance,
@@ -37,19 +38,21 @@ export const prepareCompiler = (repo: SchemaFileRepository, options: PrepareComp
   const nativeInstance = options.nativeInstance || new NativeInstance();
   const cubeDictionary = new CubeDictionary();
   const cubeSymbols = new CubeSymbols();
+  const viewCompiler = new CubeSymbols(true);
+  const viewCompilationGate = new ViewCompilationGate();
   const cubeValidator = new CubeValidator(cubeSymbols);
   const cubeEvaluator = new CubeEvaluator(cubeValidator);
   const contextEvaluator = new ContextEvaluator(cubeEvaluator);
   const joinGraph = new JoinGraph(cubeValidator, cubeEvaluator);
   const metaTransformer = new CubeToMetaTransformer(cubeValidator, cubeEvaluator, contextEvaluator, joinGraph);
   const { maxQueryCacheSize, maxQueryCacheAge } = options;
   const compilerCache = new CompilerCache({ maxQueryCacheSize, maxQueryCacheAge });
-  const yamlCompiler = new YamlCompiler(cubeSymbols, cubeDictionary, nativeInstance);
+  const yamlCompiler = new YamlCompiler(cubeSymbols, cubeDictionary, nativeInstance, viewCompiler);
 
   const transpilers: TranspilerInterface[] = [
     new ValidationTranspiler(),
     new ImportExportTranspiler(),
-    new CubePropContextTranspiler(cubeSymbols, cubeDictionary),
+    new CubePropContextTranspiler(cubeSymbols, cubeDictionary, viewCompiler),
   ];
 
   if (!options.allowJsDuplicatePropsInSchema) {
@@ -60,6 +63,8 @@ export const prepareCompiler = (repo: SchemaFileRepository, options: PrepareComp
     cubeNameCompilers: [cubeDictionary],
     preTranspileCubeCompilers: [cubeSymbols, cubeValidator],
     transpilers,
+    viewCompilationGate,
+    viewCompilers: [viewCompiler],
     cubeCompilers: [cubeEvaluator, joinGraph, metaTransformer],
     contextCompilers: [contextEvaluator],
     cubeFactory: cubeSymbols.createCube.bind(cubeSymbols),
@@ -72,7 +77,7 @@ export const prepareCompiler = (repo: SchemaFileRepository, options: PrepareComp
     compileContext: options.compileContext,
     standalone: options.standalone,
     nativeInstance,
-    yamlCompiler
+    yamlCompiler,
   }, options));
 
   return {

packages/cubejs-schema-compiler/src/compiler/ViewCompilationGate.ts

@@ -0,0 +1,30 @@
+export class ViewCompilationGate {
+  private shouldCompile: any;
+
+  public constructor() {
+    this.shouldCompile = false;
+  }
+
+  public compile(cubes: any[]) {
+    // When developing Data Access Policies feature, we've came across a
+    // limitation that Cube members can't be referenced in access policies defined on Views,
+    // because views aren't (yet) compiled at the time of access policy evaluation.
+    // To workaround this limitation and additional compilation pass is necessary,
+    // however it comes with a significant performance penalty.
+    // This gate check whether the data model contains views with access policies,
+    // and only then allows the additional compilation pass.
+    //
+    // Check out the DataSchemaCompiler.ts to see how this gate is used.
+    if (this.viewsHaveAccessPolicies(cubes)) {
+      this.shouldCompile = true;
+    }
+  }
+
+  private viewsHaveAccessPolicies(cubes: any[]) {
+    return cubes.some(c => c.isView && c.accessPolicy);
+  }
+
+  public shouldCompileViews() {
+    return this.shouldCompile;
+  }
+}

packages/cubejs-schema-compiler/src/compiler/YamlCompiler.ts

@@ -33,6 +33,7 @@ export class YamlCompiler {
     private readonly cubeSymbols: CubeSymbols,
     private readonly cubeDictionary: CubeDictionary,
     private readonly nativeInstance: NativeInstance,
+    private readonly viewCompiler: CubeSymbols,
   ) {
   }
 
@@ -288,7 +289,9 @@ export class YamlCompiler {
       },
     );
 
-    resolveSymbol = resolveSymbol || (n => this.cubeSymbols.resolveSymbol(cubeName, n) || this.cubeSymbols.isCurrentCube(n));
+    resolveSymbol = resolveSymbol || (n => this.viewCompiler.resolveSymbol(cubeName, n) ||
+      this.cubeSymbols.resolveSymbol(cubeName, n) ||
+      this.cubeSymbols.isCurrentCube(n));
 
     const traverseObj = {
       Program: (babelPath) => {

packages/cubejs-schema-compiler/src/compiler/transpilers/CubePropContextTranspiler.ts

@@ -41,6 +41,7 @@ export class CubePropContextTranspiler implements TranspilerInterface {
   public constructor(
     protected readonly cubeSymbols: CubeSymbols,
     protected readonly cubeDictionary: CubeDictionary,
+    protected readonly viewCompiler: CubeSymbols,
   ) {
   }
 
@@ -88,7 +89,9 @@ export class CubePropContextTranspiler implements TranspilerInterface {
   }
 
   protected sqlAndReferencesFieldVisitor(cubeName): TraverseObject {
-    const resolveSymbol = n => this.cubeSymbols.resolveSymbol(cubeName, n) || this.cubeSymbols.isCurrentCube(n);
+    const resolveSymbol = n => this.viewCompiler.resolveSymbol(cubeName, n) ||
+      this.cubeSymbols.resolveSymbol(cubeName, n) ||
+      this.cubeSymbols.isCurrentCube(n);
 
     return {
       ObjectProperty: (path) => {

packages/cubejs-testing/birdbox-fixtures/rbac-python/model/views/users_view.yaml

@@ -0,0 +1,13 @@
+views:
+  - name: users_view
+    cubes:
+      - join_path: users
+        includes: "*"
+
+    access_policy:
+      - role: '*'
+        row_level:
+          filters:
+            - member: id
+              operator: gt
+              values: [10]