Skip to content

Commit ca4848b

Browse files
Neko1313Neko1313
committed
feat(api-gateway): Expanding meta-information endpoints
With a large number of cubes, the json sent becomes gigantic, which creates an increased load on the network.
1 parent 7eba663 commit ca4848b

File tree

2 files changed

+245
-42
lines changed

2 files changed

+245
-42
lines changed

‎packages/cubejs-api-gateway/src/gateway.ts‎

Lines changed: 135 additions & 42 deletions
Original file line numberDiff line numberDiff line change
@@ -400,6 +400,45 @@ class ApiGateway {
400400
})
401401
);
402402

403+
app.get(
404+
`${this.basePath}/v1/meta/model_name`,
405+
userMiddlewares,
406+
userAsyncHandler(async (req, res) => {
407+
await this.metaModelName({
408+
context: req.context,
409+
res: this.resToResultFn(res),
410+
});
411+
})
412+
);
413+
414+
app.get(
415+
`${this.basePath}/v1/meta/:nameModel`,
416+
userMiddlewares,
417+
userAsyncHandler(async (req, res) => {
418+
const { nameModel } = req.params;
419+
await this.metaModelByName({
420+
context: req.context,
421+
nameModel,
422+
res: this.resToResultFn(res),
423+
});
424+
})
425+
);
426+
427+
app.get(
428+
`${this.basePath}/v1/meta/:nameModel/:field`,
429+
userMiddlewares,
430+
userAsyncHandler(async (req, res) => {
431+
const { nameModel } = req.params;
432+
const { field } = req.params;
433+
await this.metaFieldModelByName({
434+
context: req.context,
435+
nameModel,
436+
field,
437+
res: this.resToResultFn(res),
438+
});
439+
})
440+
);
441+
403442
app.post(
404443
`${this.basePath}/v1/cubesql`,
405444
userMiddlewares,
@@ -577,35 +616,54 @@ class ApiGateway {
577616
})).filter(cube => cube.config.measures?.length || cube.config.dimensions?.length || cube.config.segments?.length);
578617
}
579618

619+
private transformCubeConfig(cube: any, cubeDefinitions: any) {
620+
return {
621+
...transformCube(cube, cubeDefinitions),
622+
measures: cube.measures?.map(measure => transformMeasure(measure, cubeDefinitions)),
623+
dimensions: cube.dimensions?.map(dimension => transformDimension(dimension, cubeDefinitions)),
624+
segments: cube.segments?.map(segment => transformSegment(segment, cubeDefinitions)),
625+
joins: transformJoins(cubeDefinitions[cube.name]?.joins),
626+
preAggregations: transformPreAggregations(cubeDefinitions[cube.name]?.preAggregations)
627+
};
628+
}
629+
630+
private extractModelNames(cubes: any[]) {
631+
return cubes.map(cube => ({ name: cube.config?.name }));
632+
}
633+
634+
private async fetchMetaConfig(context: RequestContext, includeCompiler: boolean) {
635+
const compilerApi = await this.getCompilerApi(context);
636+
return compilerApi.metaConfig(context, {
637+
requestId: context.requestId,
638+
includeCompilerId: includeCompiler
639+
});
640+
}
641+
642+
private async fetchMetaConfigExtended(context: ExtendedRequestContext) {
643+
const compilerApi = await this.getCompilerApi(context);
644+
return compilerApi.metaConfigExtended(context, {
645+
requestId: context.requestId
646+
});
647+
}
648+
580649
public async meta({ context, res, includeCompilerId, onlyCompilerId }: {
581650
context: RequestContext,
582651
res: MetaResponseResultFn,
583652
includeCompilerId?: boolean,
584653
onlyCompilerId?: boolean
585654
}) {
586655
const requestStarted = new Date();
587-
588656
try {
589657
await this.assertApiScope('meta', context.securityContext);
590-
const compilerApi = await this.getCompilerApi(context);
591-
const metaConfig = await compilerApi.metaConfig(context, {
592-
requestId: context.requestId,
593-
includeCompilerId: includeCompilerId || onlyCompilerId
594-
});
658+
const metaConfig = await this.fetchMetaConfig(context, !!(includeCompilerId || onlyCompilerId));
595659
if (onlyCompilerId) {
596-
const response: { cubes: any[], compilerId?: string } = {
597-
cubes: [],
598-
compilerId: metaConfig.compilerId
599-
};
600-
res(response);
660+
res({ cubes: [], compilerId: metaConfig.compilerId });
601661
return;
602662
}
603663
const cubesConfig = includeCompilerId ? metaConfig.cubes : metaConfig;
604-
const cubes = this.filterVisibleItemsInMeta(context, cubesConfig).map(cube => cube.config);
664+
const cubes = this.filterVisibleItemsInMeta(context, cubesConfig).map((cube: any) => cube.config);
605665
const response: { cubes: any[], compilerId?: string } = { cubes };
606-
if (includeCompilerId) {
607-
response.compilerId = metaConfig.compilerId;
608-
}
666+
if (includeCompilerId) response.compilerId = metaConfig.compilerId;
609667
res(response);
610668
} catch (e: any) {
611669
this.handleError({
@@ -620,39 +678,74 @@ class ApiGateway {
620678

621679
public async metaExtended({ context, res }: { context: ExtendedRequestContext, res: ResponseResultFn }) {
622680
const requestStarted = new Date();
681+
try {
682+
await this.assertApiScope('meta', context.securityContext);
683+
const { metaConfig, cubeDefinitions } = await this.fetchMetaConfigExtended(context);
684+
const cubes = this.filterVisibleItemsInMeta(context, metaConfig)
685+
.map((meta: any) => meta.config)
686+
.map((cube: any) => this.transformCubeConfig(cube, cubeDefinitions));
687+
res({ cubes });
688+
} catch (e: any) {
689+
this.handleError({ e, context, res, requestStarted });
690+
}
691+
}
623692

693+
public async metaModelName({ context, res }: { context: ExtendedRequestContext, res: ResponseResultFn }) {
694+
const requestStarted = new Date();
624695
try {
625696
await this.assertApiScope('meta', context.securityContext);
626-
const compilerApi = await this.getCompilerApi(context);
627-
const metaConfigExtended = await compilerApi.metaConfigExtended(context, {
628-
requestId: context.requestId,
629-
});
630-
const { metaConfig, cubeDefinitions } = metaConfigExtended;
697+
const { metaConfig } = await this.fetchMetaConfigExtended(context);
698+
const cubes = this.extractModelNames(metaConfig);
699+
res({ cubes });
700+
} catch (e: any) {
701+
this.handleError({ e, context, res, requestStarted });
702+
}
703+
}
631704

632-
const cubes = this.filterVisibleItemsInMeta(context, metaConfig)
633-
.map((meta) => meta.config)
634-
.map((cube) => ({
635-
...transformCube(cube, cubeDefinitions),
636-
measures: cube.measures?.map((measure) => ({
637-
...transformMeasure(measure, cubeDefinitions),
638-
})),
639-
dimensions: cube.dimensions?.map((dimension) => ({
640-
...transformDimension(dimension, cubeDefinitions),
641-
})),
642-
segments: cube.segments?.map((segment) => ({
643-
...transformSegment(segment, cubeDefinitions),
644-
})),
645-
joins: transformJoins(cubeDefinitions[cube.name]?.joins),
646-
preAggregations: transformPreAggregations(cubeDefinitions[cube.name]?.preAggregations),
647-
}));
705+
private async getModelCubes(context: ExtendedRequestContext, nameModel: string) {
706+
const { metaConfig, cubeDefinitions } = await this.fetchMetaConfigExtended(context);
707+
return this.filterVisibleItemsInMeta(context, metaConfig)
708+
.map((meta: any) => meta.config)
709+
.map((cube: any) => this.transformCubeConfig(cube, cubeDefinitions))
710+
.filter((cube: any) => cube.name === nameModel);
711+
}
712+
713+
public async metaModelByName({ context, nameModel, res }: { context: ExtendedRequestContext, nameModel: string, res: ResponseResultFn }) {
714+
const requestStarted = new Date();
715+
try {
716+
await this.assertApiScope('meta', context.securityContext);
717+
const cubes = await this.getModelCubes(context, nameModel);
718+
if (!cubes.length) {
719+
res({ error: `Model ${nameModel} not found` });
720+
return;
721+
}
648722
res({ cubes });
649723
} catch (e: any) {
650-
this.handleError({
651-
e,
652-
context,
653-
res,
654-
requestStarted,
655-
});
724+
this.handleError({ e, context, res, requestStarted });
725+
}
726+
}
727+
728+
public async metaFieldModelByName({ context, nameModel, field, res }: { context: ExtendedRequestContext, nameModel: string, field?: string, res: ResponseResultFn }) {
729+
const requestStarted = new Date();
730+
try {
731+
await this.assertApiScope('meta', context.securityContext);
732+
const cubes = await this.getModelCubes(context, nameModel);
733+
if (!cubes.length) {
734+
res({ error: `Model ${nameModel} not found` });
735+
return;
736+
}
737+
if (field) {
738+
const cube = cubes[0];
739+
if (!(field in cube)) {
740+
res({ error: `Field ${field} not found in model ${nameModel}` });
741+
return;
742+
}
743+
res({ value: cube[field] });
744+
return;
745+
}
746+
res({ cubes });
747+
} catch (e: any) {
748+
this.handleError({ e, context, res, requestStarted });
656749
}
657750
}
658751

‎packages/cubejs-api-gateway/test/permissions.test.ts‎

Lines changed: 110 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -91,6 +91,116 @@ describe('Gateway Api Scopes', () => {
9191
apiGateway.release();
9292
});
9393

94+
test('GET /v1/meta/model_name should return model names', async () => {
95+
const { app, apiGateway } = createApiGateway({
96+
contextToApiScopes: async () => ['graphql', 'meta', 'data', 'jobs'],
97+
});
98+
99+
const response = await request(app)
100+
.get('/cubejs-api/v1/meta/model_name')
101+
.set('Authorization', AUTH_TOKEN)
102+
.expect(200);
103+
104+
expect(response.body).toEqual({ cubes: [{ name: 'Foo' }] });
105+
106+
apiGateway.release();
107+
});
108+
109+
test('GET /v1/meta/:nameModel should return model data', async () => {
110+
const { app, apiGateway } = createApiGateway({
111+
contextToApiScopes: async () => ['graphql', 'meta', 'data', 'jobs'],
112+
});
113+
114+
const response = await request(app)
115+
.get('/cubejs-api/v1/meta/Foo')
116+
.set('Authorization', AUTH_TOKEN)
117+
.expect(200);
118+
119+
expect(response.body).toEqual(
120+
{
121+
cubes: [
122+
{
123+
name: 'Foo',
124+
description: 'cube from compilerApi mock',
125+
measures: [
126+
{
127+
name: 'Foo.bar',
128+
description: 'measure from compilerApi mock',
129+
isVisible: true,
130+
},
131+
],
132+
dimensions: [
133+
{
134+
name: 'Foo.id',
135+
description: 'id dimension from compilerApi mock',
136+
isVisible: true,
137+
},
138+
{
139+
name: 'Foo.time',
140+
isVisible: true,
141+
},
142+
],
143+
segments: [
144+
{
145+
name: 'Foo.quux',
146+
description: 'segment from compilerApi mock',
147+
isVisible: true,
148+
},
149+
],
150+
sql: '\'SELECT * FROM Foo\'',
151+
}
152+
]
153+
}
154+
);
155+
156+
apiGateway.release();
157+
});
158+
159+
test('GET /v1/meta/:nameModel should return error if model not found', async () => {
160+
const { app, apiGateway } = createApiGateway({
161+
contextToApiScopes: async () => ['graphql', 'meta', 'data', 'jobs'],
162+
});
163+
164+
const response = await request(app)
165+
.get('/cubejs-api/v1/meta/UnknownModel')
166+
.set('Authorization', AUTH_TOKEN)
167+
.expect(200);
168+
169+
expect(response.body).toEqual({ error: 'Model UnknownModel not found' });
170+
171+
apiGateway.release();
172+
});
173+
174+
test('GET /v1/meta/:nameModel/:field should return specific field', async () => {
175+
const { app, apiGateway } = createApiGateway({
176+
contextToApiScopes: async () => ['graphql', 'meta', 'data', 'jobs'],
177+
});
178+
179+
const response = await request(app)
180+
.get('/cubejs-api/v1/meta/Foo/name')
181+
.set('Authorization', AUTH_TOKEN)
182+
.expect(200);
183+
184+
expect(response.body).toEqual({ value: 'Foo' });
185+
186+
apiGateway.release();
187+
});
188+
189+
test('GET /v1/meta/:nameModel/:field should return error if field not found', async () => {
190+
const { app, apiGateway } = createApiGateway({
191+
contextToApiScopes: async () => ['graphql', 'meta', 'data', 'jobs'],
192+
});
193+
194+
const response = await request(app)
195+
.get('/cubejs-api/v1/meta/Foo/unknownField')
196+
.set('Authorization', AUTH_TOKEN)
197+
.expect(200);
198+
199+
expect(response.body).toEqual({ error: 'Field unknownField not found in model Foo' });
200+
201+
apiGateway.release();
202+
});
203+
94204
test('GraphQL declined', async () => {
95205
const { app, apiGateway } = createApiGateway({
96206
contextToApiScopes: async () => ['meta', 'data', 'jobs'],

0 commit comments

Comments
 (0)