From 1487b20421b8c4afc045e8b40ca8ebccae88a18a Mon Sep 17 00:00:00 2001 From: Maxim Leonovich Date: Wed, 18 Dec 2024 10:49:44 -0800 Subject: [PATCH] fix(schema-compiler): make sure view members are resolvable in DAP --- .../src/compiler/DataSchemaCompiler.js | 7 +- .../src/compiler/PrepareCompiler.ts | 11 +- .../src/compiler/ViewCompilationGate.ts | 30 ++++ .../src/compiler/YamlCompiler.ts | 5 +- .../transpilers/CubePropContextTranspiler.ts | 5 +- .../rbac-python/model/views/users_view.yaml | 13 ++ .../rbac/model/views/users.js | 16 ++ .../__snapshots__/smoke-rbac.test.ts.snap | 150 ++++++++++++++++++ .../cubejs-testing/test/smoke-rbac.test.ts | 12 ++ yarn.lock | 2 +- 10 files changed, 244 insertions(+), 7 deletions(-) create mode 100644 packages/cubejs-schema-compiler/src/compiler/ViewCompilationGate.ts create mode 100644 packages/cubejs-testing/birdbox-fixtures/rbac-python/model/views/users_view.yaml create mode 100644 packages/cubejs-testing/birdbox-fixtures/rbac/model/views/users.js diff --git a/packages/cubejs-schema-compiler/src/compiler/DataSchemaCompiler.js b/packages/cubejs-schema-compiler/src/compiler/DataSchemaCompiler.js index 0d16153fad2d7..8ad847bd7f351 100644 --- a/packages/cubejs-schema-compiler/src/compiler/DataSchemaCompiler.js +++ b/packages/cubejs-schema-compiler/src/compiler/DataSchemaCompiler.js @@ -23,7 +23,9 @@ export class DataSchemaCompiler { this.cubeCompilers = options.cubeCompilers || []; this.contextCompilers = options.contextCompilers || []; this.transpilers = options.transpilers || []; + this.viewCompilers = options.viewCompilers || []; this.preTranspileCubeCompilers = options.preTranspileCubeCompilers || []; + this.viewCompilationGate = options.viewCompilationGate; this.cubeNameCompilers = options.cubeNameCompilers || []; this.extensions = options.extensions || {}; this.cubeFactory = options.cubeFactory; @@ -93,7 +95,10 @@ export class DataSchemaCompiler { const compilePhase = (compilers) => this.compileCubeFiles(compilers, transpile(), errorsReport); return compilePhase({ cubeCompilers: this.cubeNameCompilers }) - .then(() => compilePhase({ cubeCompilers: this.preTranspileCubeCompilers })) + .then(() => compilePhase({ cubeCompilers: this.preTranspileCubeCompilers.concat([this.viewCompilationGate]) })) + .then(() => (this.viewCompilationGate.shouldCompileViews() ? + compilePhase({ cubeCompilers: this.viewCompilers }) + : Promise.resolve())) .then(() => compilePhase({ cubeCompilers: this.cubeCompilers, contextCompilers: this.contextCompilers, diff --git a/packages/cubejs-schema-compiler/src/compiler/PrepareCompiler.ts b/packages/cubejs-schema-compiler/src/compiler/PrepareCompiler.ts index 446769688f07d..9605b1dac3d1e 100644 --- a/packages/cubejs-schema-compiler/src/compiler/PrepareCompiler.ts +++ b/packages/cubejs-schema-compiler/src/compiler/PrepareCompiler.ts @@ -20,6 +20,7 @@ import { JoinGraph } from './JoinGraph'; import { CubeToMetaTransformer } from './CubeToMetaTransformer'; import { CompilerCache } from './CompilerCache'; import { YamlCompiler } from './YamlCompiler'; +import { ViewCompilationGate } from './ViewCompilationGate'; export type PrepareCompilerOptions = { nativeInstance?: NativeInstance, @@ -37,6 +38,8 @@ export const prepareCompiler = (repo: SchemaFileRepository, options: PrepareComp const nativeInstance = options.nativeInstance || new NativeInstance(); const cubeDictionary = new CubeDictionary(); const cubeSymbols = new CubeSymbols(); + const viewCompiler = new CubeSymbols(true); + const viewCompilationGate = new ViewCompilationGate(); const cubeValidator = new CubeValidator(cubeSymbols); const cubeEvaluator = new CubeEvaluator(cubeValidator); const contextEvaluator = new ContextEvaluator(cubeEvaluator); @@ -44,12 +47,12 @@ export const prepareCompiler = (repo: SchemaFileRepository, options: PrepareComp const metaTransformer = new CubeToMetaTransformer(cubeValidator, cubeEvaluator, contextEvaluator, joinGraph); const { maxQueryCacheSize, maxQueryCacheAge } = options; const compilerCache = new CompilerCache({ maxQueryCacheSize, maxQueryCacheAge }); - const yamlCompiler = new YamlCompiler(cubeSymbols, cubeDictionary, nativeInstance); + const yamlCompiler = new YamlCompiler(cubeSymbols, cubeDictionary, nativeInstance, viewCompiler); const transpilers: TranspilerInterface[] = [ new ValidationTranspiler(), new ImportExportTranspiler(), - new CubePropContextTranspiler(cubeSymbols, cubeDictionary), + new CubePropContextTranspiler(cubeSymbols, cubeDictionary, viewCompiler), ]; if (!options.allowJsDuplicatePropsInSchema) { @@ -60,6 +63,8 @@ export const prepareCompiler = (repo: SchemaFileRepository, options: PrepareComp cubeNameCompilers: [cubeDictionary], preTranspileCubeCompilers: [cubeSymbols, cubeValidator], transpilers, + viewCompilationGate, + viewCompilers: [viewCompiler], cubeCompilers: [cubeEvaluator, joinGraph, metaTransformer], contextCompilers: [contextEvaluator], cubeFactory: cubeSymbols.createCube.bind(cubeSymbols), @@ -72,7 +77,7 @@ export const prepareCompiler = (repo: SchemaFileRepository, options: PrepareComp compileContext: options.compileContext, standalone: options.standalone, nativeInstance, - yamlCompiler + yamlCompiler, }, options)); return { diff --git a/packages/cubejs-schema-compiler/src/compiler/ViewCompilationGate.ts b/packages/cubejs-schema-compiler/src/compiler/ViewCompilationGate.ts new file mode 100644 index 0000000000000..af53f12f30de6 --- /dev/null +++ b/packages/cubejs-schema-compiler/src/compiler/ViewCompilationGate.ts @@ -0,0 +1,30 @@ +export class ViewCompilationGate { + private shouldCompile: any; + + public constructor() { + this.shouldCompile = false; + } + + public compile(cubes: any[]) { + // When developing Data Access Policies feature, we've came across a + // limitation that Cube members can't be referenced in access policies defined on Views, + // because views aren't (yet) compiled at the time of access policy evaluation. + // To workaround this limitation and additional compilation pass is necessary, + // however it comes with a significant performance penalty. + // This gate check whether the data model contains views with access policies, + // and only then allows the additional compilation pass. + // + // Check out the DataSchemaCompiler.ts to see how this gate is used. + if (this.viewsHaveAccessPolicies(cubes)) { + this.shouldCompile = true; + } + } + + private viewsHaveAccessPolicies(cubes: any[]) { + return cubes.some(c => c.isView && c.accessPolicy); + } + + public shouldCompileViews() { + return this.shouldCompile; + } +} diff --git a/packages/cubejs-schema-compiler/src/compiler/YamlCompiler.ts b/packages/cubejs-schema-compiler/src/compiler/YamlCompiler.ts index 3ac750737d743..30933af64df3d 100644 --- a/packages/cubejs-schema-compiler/src/compiler/YamlCompiler.ts +++ b/packages/cubejs-schema-compiler/src/compiler/YamlCompiler.ts @@ -33,6 +33,7 @@ export class YamlCompiler { private readonly cubeSymbols: CubeSymbols, private readonly cubeDictionary: CubeDictionary, private readonly nativeInstance: NativeInstance, + private readonly viewCompiler: CubeSymbols, ) { } @@ -288,7 +289,9 @@ export class YamlCompiler { }, ); - resolveSymbol = resolveSymbol || (n => this.cubeSymbols.resolveSymbol(cubeName, n) || this.cubeSymbols.isCurrentCube(n)); + resolveSymbol = resolveSymbol || (n => this.viewCompiler.resolveSymbol(cubeName, n) || + this.cubeSymbols.resolveSymbol(cubeName, n) || + this.cubeSymbols.isCurrentCube(n)); const traverseObj = { Program: (babelPath) => { diff --git a/packages/cubejs-schema-compiler/src/compiler/transpilers/CubePropContextTranspiler.ts b/packages/cubejs-schema-compiler/src/compiler/transpilers/CubePropContextTranspiler.ts index e0ece55a883a7..2f2574f2d8ed9 100644 --- a/packages/cubejs-schema-compiler/src/compiler/transpilers/CubePropContextTranspiler.ts +++ b/packages/cubejs-schema-compiler/src/compiler/transpilers/CubePropContextTranspiler.ts @@ -41,6 +41,7 @@ export class CubePropContextTranspiler implements TranspilerInterface { public constructor( protected readonly cubeSymbols: CubeSymbols, protected readonly cubeDictionary: CubeDictionary, + protected readonly viewCompiler: CubeSymbols, ) { } @@ -88,7 +89,9 @@ export class CubePropContextTranspiler implements TranspilerInterface { } protected sqlAndReferencesFieldVisitor(cubeName): TraverseObject { - const resolveSymbol = n => this.cubeSymbols.resolveSymbol(cubeName, n) || this.cubeSymbols.isCurrentCube(n); + const resolveSymbol = n => this.viewCompiler.resolveSymbol(cubeName, n) || + this.cubeSymbols.resolveSymbol(cubeName, n) || + this.cubeSymbols.isCurrentCube(n); return { ObjectProperty: (path) => { diff --git a/packages/cubejs-testing/birdbox-fixtures/rbac-python/model/views/users_view.yaml b/packages/cubejs-testing/birdbox-fixtures/rbac-python/model/views/users_view.yaml new file mode 100644 index 0000000000000..0163cc4e83681 --- /dev/null +++ b/packages/cubejs-testing/birdbox-fixtures/rbac-python/model/views/users_view.yaml @@ -0,0 +1,13 @@ +views: + - name: users_view + cubes: + - join_path: users + includes: "*" + + access_policy: + - role: '*' + row_level: + filters: + - member: id + operator: gt + values: [10] diff --git a/packages/cubejs-testing/birdbox-fixtures/rbac/model/views/users.js b/packages/cubejs-testing/birdbox-fixtures/rbac/model/views/users.js new file mode 100644 index 0000000000000..1c9488769f092 --- /dev/null +++ b/packages/cubejs-testing/birdbox-fixtures/rbac/model/views/users.js @@ -0,0 +1,16 @@ +view('users_view', { + cubes: [{ + join_path: users, + includes: '*', + }], + accessPolicy: [{ + role: '*', + rowLevel: { + filters: [{ + member: 'id', + operator: 'gt', + values: [10], + }], + }, + }] +}); diff --git a/packages/cubejs-testing/test/__snapshots__/smoke-rbac.test.ts.snap b/packages/cubejs-testing/test/__snapshots__/smoke-rbac.test.ts.snap index c6fd88b4ced72..b4a7ce5537530 100644 --- a/packages/cubejs-testing/test/__snapshots__/smoke-rbac.test.ts.snap +++ b/packages/cubejs-testing/test/__snapshots__/smoke-rbac.test.ts.snap @@ -8,6 +8,81 @@ Array [ ] `; +exports[`Cube RBAC Engine [Python config] RBAC via SQL API [python config] SELECT * from users_view: users_view_python 1`] = ` +Array [ + Object { + "__cubeJoinField": null, + "__user": null, + "city": "Austin", + "count": "1", + "id": 400, + }, + Object { + "__cubeJoinField": null, + "__user": null, + "city": "Austin", + "count": "1", + "id": 401, + }, + Object { + "__cubeJoinField": null, + "__user": null, + "city": "Austin", + "count": "1", + "id": 402, + }, + Object { + "__cubeJoinField": null, + "__user": null, + "city": "Austin", + "count": "1", + "id": 403, + }, + Object { + "__cubeJoinField": null, + "__user": null, + "city": "Austin", + "count": "1", + "id": 404, + }, + Object { + "__cubeJoinField": null, + "__user": null, + "city": "Austin", + "count": "1", + "id": 405, + }, + Object { + "__cubeJoinField": null, + "__user": null, + "city": "Austin", + "count": "1", + "id": 406, + }, + Object { + "__cubeJoinField": null, + "__user": null, + "city": "Austin", + "count": "1", + "id": 407, + }, + Object { + "__cubeJoinField": null, + "__user": null, + "city": "Austin", + "count": "1", + "id": 408, + }, + Object { + "__cubeJoinField": null, + "__user": null, + "city": "Austin", + "count": "1", + "id": 409, + }, +] +`; + exports[`Cube RBAC Engine [Python config][dev mode] products with no matching policy: products_no_policy_python 1`] = ` Array [ Object { @@ -611,6 +686,81 @@ Array [ ] `; +exports[`Cube RBAC Engine RBAC via SQL API SELECT * from users_view: users_view_js 1`] = ` +Array [ + Object { + "__cubeJoinField": null, + "__user": null, + "city": "Austin", + "count": "1", + "id": 400, + }, + Object { + "__cubeJoinField": null, + "__user": null, + "city": "Austin", + "count": "1", + "id": 401, + }, + Object { + "__cubeJoinField": null, + "__user": null, + "city": "Austin", + "count": "1", + "id": 402, + }, + Object { + "__cubeJoinField": null, + "__user": null, + "city": "Austin", + "count": "1", + "id": 403, + }, + Object { + "__cubeJoinField": null, + "__user": null, + "city": "Austin", + "count": "1", + "id": 404, + }, + Object { + "__cubeJoinField": null, + "__user": null, + "city": "Austin", + "count": "1", + "id": 405, + }, + Object { + "__cubeJoinField": null, + "__user": null, + "city": "Austin", + "count": "1", + "id": 406, + }, + Object { + "__cubeJoinField": null, + "__user": null, + "city": "Austin", + "count": "1", + "id": 407, + }, + Object { + "__cubeJoinField": null, + "__user": null, + "city": "Austin", + "count": "1", + "id": 408, + }, + Object { + "__cubeJoinField": null, + "__user": null, + "city": "Austin", + "count": "1", + "id": 409, + }, +] +`; + exports[`Cube RBAC Engine RBAC via SQL API default policy SELECT with member expressions: users_member_expression 1`] = ` Array [ Object { diff --git a/packages/cubejs-testing/test/smoke-rbac.test.ts b/packages/cubejs-testing/test/smoke-rbac.test.ts index 56823762e3eb5..f318711932a8c 100644 --- a/packages/cubejs-testing/test/smoke-rbac.test.ts +++ b/packages/cubejs-testing/test/smoke-rbac.test.ts @@ -152,6 +152,12 @@ describe('Cube RBAC Engine', () => { // Querying a cube with nested filters and mixed values should not cause any issues expect(res.rows).toMatchSnapshot('users'); }); + + test('SELECT * from users_view', async () => { + const res = await connection.query('SELECT * FROM users_view limit 10'); + // Make sure view policies are evaluated correctly in yaml schemas + expect(res.rows).toMatchSnapshot('users_view_js'); + }); }); describe('RBAC via SQL API manager', () => { @@ -398,6 +404,12 @@ describe('Cube RBAC Engine [Python config]', () => { // It should also exclude the `created_at` dimension as per memberLevel policy expect(res.rows).toMatchSnapshot('users_python'); }); + + test('SELECT * from users_view', async () => { + const res = await connection.query('SELECT * FROM users_view limit 10'); + // Make sure view policies are evaluated correctly in yaml schemas + expect(res.rows).toMatchSnapshot('users_view_python'); + }); }); }); diff --git a/yarn.lock b/yarn.lock index 97afdc40554b4..474cc1bac6450 100644 --- a/yarn.lock +++ b/yarn.lock @@ -9890,7 +9890,7 @@ dependencies: "@types/node" "*" -"@types/node@*", "@types/node@^12", "@types/node@^16", "@types/node@^18": +"@types/node@*", "@types/node@^12", "@types/node@^18": version "18.19.46" resolved "https://registry.yarnpkg.com/@types/node/-/node-18.19.46.tgz#51801396c01153e0626e36f43386e83bc768b072" integrity sha512-vnRgMS7W6cKa1/0G3/DTtQYpVrZ8c0Xm6UkLaVFrb9jtcVC3okokW09Ki1Qdrj9ISokszD69nY4WDLRlvHlhAA==