From 770b6627cd0e4dc1dfd21dca72bcc8a33c180f2f Mon Sep 17 00:00:00 2001 From: Konstantin Burkalev Date: Thu, 28 Nov 2024 17:10:23 +0200 Subject: [PATCH 01/16] =?UTF-8?q?feat(server-core):=20Support=20for=C2=A0s?= =?UTF-8?q?cheduledRefreshTimeZones=20as=20function,=20passing=20securityC?= =?UTF-8?q?ontext?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- packages/cubejs-api-gateway/src/gateway.ts | 6 +-- .../cubejs-api-gateway/src/types/gateway.ts | 13 +++++- .../cubejs-api-gateway/test/index.test.ts | 2 +- .../cubejs-server-core/src/core/server.ts | 41 +++++++++++++++---- packages/cubejs-server-core/src/core/types.ts | 12 +++++- 5 files changed, 58 insertions(+), 16 deletions(-) diff --git a/packages/cubejs-api-gateway/src/gateway.ts b/packages/cubejs-api-gateway/src/gateway.ts index 6737ca6fef35c..0be2de9192d18 100644 --- a/packages/cubejs-api-gateway/src/gateway.ts +++ b/packages/cubejs-api-gateway/src/gateway.ts @@ -456,7 +456,7 @@ class ApiGateway { app.get('/cubejs-system/v1/pre-aggregations/timezones', systemMiddlewares, systemAsyncHandler(async (req, res) => { this.resToResultFn(res)({ - timezones: this.scheduledRefreshTimeZones || [] + timezones: this.scheduledRefreshTimeZones ? this.scheduledRefreshTimeZones(req.context) : [] }); })); @@ -630,7 +630,7 @@ class ApiGateway { context, normalizeQueryPreAggregations( { - timezones: this.scheduledRefreshTimeZones, + timezones: this.scheduledRefreshTimeZones ? this.scheduledRefreshTimeZones(context) : [], preAggregations: preAggregations.map(p => ({ id: p.id, cacheOnly, @@ -654,7 +654,7 @@ class ApiGateway { try { query = normalizeQueryPreAggregations( this.parseQueryParam(query), - { timezones: this.scheduledRefreshTimeZones } + { timezones: this.scheduledRefreshTimeZones ? this.scheduledRefreshTimeZones(context) : [] } ); const orchestratorApi = await this.getAdapterApi(context); const compilerApi = await this.getCompilerApi(context); diff --git a/packages/cubejs-api-gateway/src/types/gateway.ts b/packages/cubejs-api-gateway/src/types/gateway.ts index 352cc61b4ee3d..e05f22d3c1e47 100644 --- a/packages/cubejs-api-gateway/src/types/gateway.ts +++ b/packages/cubejs-api-gateway/src/types/gateway.ts @@ -34,8 +34,15 @@ type UserBackgroundContext = { authInfo?: any; }; +type RequestContext = { + // @deprecated Renamed to securityContext, please use securityContext. + authInfo?: any; + securityContext: any; + requestId: string; +}; + /** - * Function that should provides a logic of scheduled returning of + * Function that should provide a logic of scheduled returning of * the user background context. Used as a part of a main * configuration object of the Gateway to provide extendability to * this logic. @@ -43,6 +50,8 @@ type UserBackgroundContext = { type ScheduledRefreshContextsFn = () => Promise; +type ScheduledRefreshTimeZonesFn = (context: RequestContext) => string[] | Promise; + /** * Gateway configuration options interface. */ @@ -52,7 +61,7 @@ interface ApiGatewayOptions { dataSourceStorage: any; refreshScheduler: any; scheduledRefreshContexts?: ScheduledRefreshContextsFn; - scheduledRefreshTimeZones?: String[]; + scheduledRefreshTimeZones?: ScheduledRefreshTimeZonesFn; basePath: string; extendContext?: ExtendContextFn; jwt?: JWTOptions; diff --git a/packages/cubejs-api-gateway/test/index.test.ts b/packages/cubejs-api-gateway/test/index.test.ts index 658774e98919c..bdff7907ee703 100644 --- a/packages/cubejs-api-gateway/test/index.test.ts +++ b/packages/cubejs-api-gateway/test/index.test.ts @@ -849,7 +849,7 @@ describe('API Gateway', () => { playgroundAuthSecret, refreshScheduler: () => new RefreshSchedulerMock(), scheduledRefreshContexts: () => Promise.resolve(scheduledRefreshContextsFactory()), - scheduledRefreshTimeZones: scheduledRefreshTimeZonesFactory() + scheduledRefreshTimeZones: scheduledRefreshTimeZonesFactory } ); const token = generateAuthToken({ uid: 5, scope }, {}, playgroundAuthSecret); diff --git a/packages/cubejs-server-core/src/core/server.ts b/packages/cubejs-server-core/src/core/server.ts index 673096142a720..d891394dbff13 100644 --- a/packages/cubejs-server-core/src/core/server.ts +++ b/packages/cubejs-server-core/src/core/server.ts @@ -5,11 +5,23 @@ import LRUCache from 'lru-cache'; import isDocker from 'is-docker'; import pLimit from 'p-limit'; -import { ApiGateway, ApiGatewayOptions, UserBackgroundContext } from '@cubejs-backend/api-gateway'; +import { + ApiGateway, + ApiGatewayOptions, + UserBackgroundContext +} from '@cubejs-backend/api-gateway'; import { CancelableInterval, - createCancelableInterval, formatDuration, getAnonymousId, - getEnv, assertDataSource, getRealType, internalExceptions, track, FileRepository, SchemaFileRepository, + createCancelableInterval, + formatDuration, + getAnonymousId, + getEnv, + assertDataSource, + getRealType, + internalExceptions, + track, + FileRepository, + SchemaFileRepository, } from '@cubejs-backend/shared'; import type { Application as ExpressApplication } from 'express'; @@ -46,8 +58,15 @@ import type { DriverContext, LoggerFn, DriverConfig, + ScheduledRefreshTimeZonesFn, +} from './types'; +import { + ContextToOrchestratorIdFn, + ContextAcceptanceResult, + ContextAcceptanceResultHttp, + ContextAcceptanceResultWs, + ContextAcceptor } from './types'; -import { ContextToOrchestratorIdFn, ContextAcceptanceResult, ContextAcceptanceResultHttp, ContextAcceptanceResultWs, ContextAcceptor } from './types'; const { version } = require('../../../package.json'); @@ -119,6 +138,8 @@ export class CubejsServerCore { protected readonly preAggregationsSchema: PreAggregationsSchemaFn; + protected readonly scheduledRefreshTimeZones: ScheduledRefreshTimeZonesFn; + protected readonly orchestratorOptions: OrchestratorOptionsFn; public logger: LoggerFn; @@ -173,6 +194,7 @@ export class CubejsServerCore { this.contextToExternalDbType = wrapToFnIfNeeded(this.options.externalDbType); this.preAggregationsSchema = wrapToFnIfNeeded(this.options.preAggregationsSchema); this.orchestratorOptions = wrapToFnIfNeeded(this.options.orchestratorOptions); + this.scheduledRefreshTimeZones = wrapToFnIfNeeded(this.options.scheduledRefreshTimeZones || []); this.compilerCache = new LRUCache({ max: this.options.compilerCacheSize || 250, @@ -453,7 +475,7 @@ export class CubejsServerCore { jwt: this.options.jwt, refreshScheduler: this.getRefreshScheduler.bind(this), scheduledRefreshContexts: this.options.scheduledRefreshContexts, - scheduledRefreshTimeZones: this.options.scheduledRefreshTimeZones, + scheduledRefreshTimeZones: this.scheduledRefreshTimeZones, serverCoreVersion: this.coreServerVersion, contextToApiScopes: this.options.contextToApiScopes, gatewayPort: this.options.gatewayPort, @@ -700,7 +722,7 @@ export class CubejsServerCore { } /** - * @internal Please dont use this method directly, use refreshTimer + * @internal Please don't use this method directly, use refreshTimer */ public handleScheduledRefreshInterval = async (options) => { const allContexts = await this.options.scheduledRefreshContexts(); @@ -730,8 +752,9 @@ export class CubejsServerCore { concurrency: this.options.scheduledRefreshConcurrency, }; - if (this.options.scheduledRefreshTimeZones) { - queryingOptions.timezones = this.options.scheduledRefreshTimeZones; + const timezonesFromOptionsOrSecurityContext = await this.scheduledRefreshTimeZones(context); + if (timezonesFromOptionsOrSecurityContext.length > 0) { + queryingOptions.timezones = timezonesFromOptionsOrSecurityContext; } return this.runScheduledRefresh(context, queryingOptions); @@ -746,7 +769,7 @@ export class CubejsServerCore { } /** - * @internal Please dont use this method directly, use refreshTimer + * @internal Please don't use this method directly, use refreshTimer */ public async runScheduledRefresh(context: UserBackgroundContext | null, queryingOptions?: ScheduledRefreshOptions) { return this.getRefreshScheduler().runScheduledRefresh( diff --git a/packages/cubejs-server-core/src/core/types.ts b/packages/cubejs-server-core/src/core/types.ts index a2ea43048df50..d639e2664cf2e 100644 --- a/packages/cubejs-server-core/src/core/types.ts +++ b/packages/cubejs-server-core/src/core/types.ts @@ -128,6 +128,16 @@ export type OrchestratorOptionsFn = (context: RequestContext) => OrchestratorOpt export type PreAggregationsSchemaFn = (context: RequestContext) => string | Promise; +export type ScheduledRefreshTimeZonesFn = (context: RequestContext) => string[] | Promise; + +/** + * Function that should provide a logic of scheduled returning of + * the user background context. Used as a part of a main + * configuration object of the Gateway to provide extendability to + * this logic. + */ +export type ScheduledRefreshContextsFn = () => Promise; + // internal export type DriverOptions = { dataSource?: string, @@ -194,7 +204,7 @@ export interface CreateOptions { schemaVersion?: (context: RequestContext) => string | Promise; extendContext?: ExtendContextFn; scheduledRefreshTimer?: boolean | number; - scheduledRefreshTimeZones?: string[]; + scheduledRefreshTimeZones?: string[] | ScheduledRefreshTimeZonesFn; scheduledRefreshContexts?: () => Promise; scheduledRefreshConcurrency?: number; scheduledRefreshBatchSize?: number; From 696d6af3d7c5ae2fdebe815daa5018adb8f08243 Mon Sep 17 00:00:00 2001 From: Konstantin Burkalev Date: Thu, 28 Nov 2024 17:53:17 +0200 Subject: [PATCH 02/16] add scheduledRefreshTimeZones schema validation --- packages/cubejs-server-core/src/core/optionsValidate.ts | 5 ++++- packages/cubejs-server-core/test/unit/OptsHandler.test.ts | 4 ++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/packages/cubejs-server-core/src/core/optionsValidate.ts b/packages/cubejs-server-core/src/core/optionsValidate.ts index 157d2acb84cd9..1cf479adee4bc 100644 --- a/packages/cubejs-server-core/src/core/optionsValidate.ts +++ b/packages/cubejs-server-core/src/core/optionsValidate.ts @@ -92,7 +92,10 @@ const schemaOptions = Joi.object().keys({ Joi.boolean(), Joi.number().min(0).integer() ), - scheduledRefreshTimeZones: Joi.array().items(Joi.string()), + scheduledRefreshTimeZones: Joi.alternatives().try( + Joi.array().items(Joi.string()), + Joi.func() + ), scheduledRefreshContexts: Joi.func(), scheduledRefreshConcurrency: Joi.number().min(1).integer(), scheduledRefreshBatchSize: Joi.number().min(1).integer(), diff --git a/packages/cubejs-server-core/test/unit/OptsHandler.test.ts b/packages/cubejs-server-core/test/unit/OptsHandler.test.ts index 5066819a9235b..3d37757679c8b 100644 --- a/packages/cubejs-server-core/test/unit/OptsHandler.test.ts +++ b/packages/cubejs-server-core/test/unit/OptsHandler.test.ts @@ -76,7 +76,7 @@ describe('OptsHandler class', () => { } ); - test('must handle vanila CreateOptions', async () => { + test('must handle vanilla CreateOptions', async () => { process.env.CUBEJS_DB_TYPE = 'postgres'; // Case 1 @@ -432,7 +432,7 @@ describe('OptsHandler class', () => { testDriverConnectionSpy.mockRestore(); }); - test('must determine correcct driver type by the query context', async () => { + test('must determine correct driver type by the query context', async () => { class Driver1 extends OriginalBaseDriver { public async testConnection() { // From 531c3508d1011b943541c24b67aef357f9152ca0 Mon Sep 17 00:00:00 2001 From: Konstantin Burkalev Date: Thu, 28 Nov 2024 19:53:05 +0200 Subject: [PATCH 03/16] fix context passing --- packages/cubejs-server-core/src/core/server.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/packages/cubejs-server-core/src/core/server.ts b/packages/cubejs-server-core/src/core/server.ts index d891394dbff13..6f226eac19331 100644 --- a/packages/cubejs-server-core/src/core/server.ts +++ b/packages/cubejs-server-core/src/core/server.ts @@ -735,11 +735,11 @@ export class CubejsServerCore { const contexts = []; for (const allContext of allContexts) { - const res = await this.contextAcceptor.shouldAccept( - this.migrateBackgroundContext(allContext) - ); + const resContext = this.migrateBackgroundContext(allContext); + const res = await this.contextAcceptor.shouldAccept(resContext); + if (res.accepted) { - contexts.push(allContext); + contexts.push(resContext || {}); } } From 2d4e6dad560e0029eb160a54c4e039b30460cf4c Mon Sep 17 00:00:00 2001 From: Konstantin Burkalev Date: Thu, 28 Nov 2024 19:53:40 +0200 Subject: [PATCH 04/16] add tests --- .../test/unit/index.test.ts | 75 +++++++++++++++++++ 1 file changed, 75 insertions(+) diff --git a/packages/cubejs-server-core/test/unit/index.test.ts b/packages/cubejs-server-core/test/unit/index.test.ts index a5451db723a02..a1f1741d38303 100644 --- a/packages/cubejs-server-core/test/unit/index.test.ts +++ b/packages/cubejs-server-core/test/unit/index.test.ts @@ -833,4 +833,79 @@ describe('index.test', () => { jest.restoreAllMocks(); }); + + test('scheduledRefreshTimeZones option', async () => { + jest.spyOn( + CubejsServerCoreOpen.prototype, + 'isReadyForQueryProcessing', + ).mockImplementation( + () => true, + ); + + const timeoutKiller = withTimeout( + () => { + throw new Error('scheduledRefreshTimeZones was not called'); + }, + 3 * 1000, + ); + + let counter = 0; + + const cubejsServerCore = new CubejsServerCoreOpen({ + dbType: 'mysql', + apiSecret: 'secret', + // 250ms + scheduledRefreshTimer: 1, + scheduledRefreshConcurrency: 1, + scheduledRefreshContexts: async () => [ + { + securityContext: { + appid: 'test1', + u: { + prop1: 'value1' + } + } + }, + // securityContext is required in typings, but can be empty in user-space + { + // Renamed to securityContext, let's test that it migrate automatically + authInfo: { + appid: 'test2', + u: { + prop2: 'value2' + } + }, + }, + // Null is a default placeholder + null + ], + scheduledRefreshTimeZones: async (ctx) => { + counter++; + if (counter === 1) { + expect(ctx.securityContext).toEqual({ appid: 'test1', u: { prop1: 'value1' } }); + return ['Europe/Kyiv']; + } else if (counter === 2) { + expect(ctx.securityContext).toEqual({ appid: 'test2', u: { prop2: 'value2' } }); + return ['Europe/London']; + } else if (counter === 3) { + expect(ctx.securityContext).toEqual(undefined); + + // Kill the timer after processing all 3 test contexts + await timeoutKiller.cancel(); + + return ['America/Los_Angeles']; + } + + return ['Europe/Kyiv', 'Europe/London', 'America/Los_Angeles']; + } + }); + + await timeoutKiller; + + expect(cubejsServerCore).toBeInstanceOf(CubejsServerCoreOpen); + expect(counter).toBe(3); + + await cubejsServerCore.beforeShutdown(); + await cubejsServerCore.shutdown(); + }); }); From 68dc63e18c669b4566ba0f8dcc1abdf0524a8552 Mon Sep 17 00:00:00 2001 From: Konstantin Burkalev Date: Thu, 28 Nov 2024 21:10:26 +0200 Subject: [PATCH 05/16] fix --- packages/cubejs-api-gateway/src/gateway.ts | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/packages/cubejs-api-gateway/src/gateway.ts b/packages/cubejs-api-gateway/src/gateway.ts index 0be2de9192d18..e76a8d5678cc4 100644 --- a/packages/cubejs-api-gateway/src/gateway.ts +++ b/packages/cubejs-api-gateway/src/gateway.ts @@ -625,12 +625,13 @@ class ApiGateway { const compilerApi = await this.getCompilerApi(context); const preAggregations = await compilerApi.preAggregations(); + const refreshTimezones = this.scheduledRefreshTimeZones ? await this.scheduledRefreshTimeZones(context) : []; const preAggregationPartitions = await this.refreshScheduler() .preAggregationPartitions( context, normalizeQueryPreAggregations( { - timezones: this.scheduledRefreshTimeZones ? this.scheduledRefreshTimeZones(context) : [], + timezones: refreshTimezones.length > 0 ? refreshTimezones : undefined, preAggregations: preAggregations.map(p => ({ id: p.id, cacheOnly, @@ -652,9 +653,10 @@ class ApiGateway { ) { const requestStarted = new Date(); try { + const refreshTimezones = this.scheduledRefreshTimeZones ? await this.scheduledRefreshTimeZones(context) : []; query = normalizeQueryPreAggregations( this.parseQueryParam(query), - { timezones: this.scheduledRefreshTimeZones ? this.scheduledRefreshTimeZones(context) : [] } + { timezones: refreshTimezones.length > 0 ? refreshTimezones : undefined } ); const orchestratorApi = await this.getAdapterApi(context); const compilerApi = await this.getCompilerApi(context); From 29320aa8006a7b773472bb1be1b645120d0e1392 Mon Sep 17 00:00:00 2001 From: Konstantin Burkalev Date: Thu, 28 Nov 2024 21:45:33 +0200 Subject: [PATCH 06/16] fix lint warnings --- packages/cubejs-api-gateway/test/auth.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/cubejs-api-gateway/test/auth.test.ts b/packages/cubejs-api-gateway/test/auth.test.ts index cbb728f60c248..fe52541d2b172 100644 --- a/packages/cubejs-api-gateway/test/auth.test.ts +++ b/packages/cubejs-api-gateway/test/auth.test.ts @@ -323,7 +323,7 @@ describe('test authorization', () => { const { app } = createApiGateway(handlerMock, loggerMock, { playgroundAuthSecret, - checkAuth: async (req: Request, auth?: string) => { + checkAuth: async (_req: Request, _auth?: string) => { throw new CubejsHandlerError(409, 'Error', 'Custom error'); } }); From 5bc434083dc396680a0098774db8d2fcf633721f Mon Sep 17 00:00:00 2001 From: Igor Lukanin Date: Mon, 2 Dec 2024 13:35:49 +0100 Subject: [PATCH 07/16] docs: Cross-links between env vars and config options + clean-up --- docs/pages/reference/configuration/config.mdx | 87 +++++++++---------- .../configuration/environment-variables.mdx | 76 ++++++++++++++-- 2 files changed, 110 insertions(+), 53 deletions(-) diff --git a/docs/pages/reference/configuration/config.mdx b/docs/pages/reference/configuration/config.mdx index 4757bd4638ccd..886207887fa25 100644 --- a/docs/pages/reference/configuration/config.mdx +++ b/docs/pages/reference/configuration/config.mdx @@ -46,7 +46,8 @@ module.exports = { -Overrides `CUBEJS_SCHEMA_PATH`. The default value is `model`. +This configuration option can also be set using the `CUBEJS_SCHEMA_PATH` +environment variable. The default value is `model`. Use [`repositoryFactory`][self-repofactory] for [multitenancy][ref-multitenancy] or when a more flexible setup is needed. @@ -309,6 +310,9 @@ module.exports = { +This configuration option can also be set using the `CUBEJS_CACHE_AND_QUEUE_DRIVER` +environment variable. + ### `context_to_orchestrator_id` @@ -447,18 +451,18 @@ these values can result in application instability and/or downtime. You can pass this object to set advanced options for the query orchestrator. -| Option | Description | Default Value | -|----------------------------------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|-------------------------| -| continueWaitTimeout | Long polling interval in seconds, maximum is 90 | `5` | -| rollupOnlyMode | When enabled, an error will be thrown if a query can't be served from a pre-aggregation (rollup) | `false` | -| queryCacheOptions | Query cache options for DB queries | `{}` | -| queryCacheOptions.refreshKeyRenewalThreshold | Time in seconds to cache the result of [`refresh_key`][ref-schema-cube-ref-refresh-key] check | `defined by DB dialect` | -| queryCacheOptions.backgroundRenew | Controls whether to wait in foreground for refreshed query data if `refresh_key` value has been changed. Refresh key queries or pre-aggregations are never awaited in foreground and always processed in background unless cache is empty. If `true` it immediately returns values from cache if available without [`refresh_key`][ref-schema-cube-ref-refresh-key] check to renew in foreground. | `false` | -| queryCacheOptions.queueOptions | Query queue options for DB queries | `{}` | -| preAggregationsOptions | Query cache options for pre-aggregations | `{}` | -| preAggregationsOptions.maxPartitions | The maximum number of partitions each pre-aggregation in a cube can use. | `10000` | -| preAggregationsOptions.queueOptions | Query queue options for pre-aggregations | `{}` | -| preAggregationsOptions.externalRefresh | When running a separate instance of Cube to refresh pre-aggregations in the background, this option can be set on the API instance to prevent it from trying to check for rollup data being current - it won't try to create or refresh them when this option is `true` | `false` | +| Option | Description | Default Value | +|------------------------------------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|-------------------------| +| `continueWaitTimeout` | Long polling interval in seconds, maximum is 90 | `5` | +| `rollupOnlyMode` | When enabled, an error will be thrown if a query can't be served from a pre-aggregation (rollup) | `false` | +| `queryCacheOptions` | Query cache options for DB queries | `{}` | +| `queryCacheOptions.refreshKeyRenewalThreshold` | Time in seconds to cache the result of [`refresh_key`][ref-schema-cube-ref-refresh-key] check | `defined by DB dialect` | +| `queryCacheOptions.backgroundRenew` | Controls whether to wait in foreground for refreshed query data if `refresh_key` value has been changed. Refresh key queries or pre-aggregations are never awaited in foreground and always processed in background unless cache is empty. If `true` it immediately returns values from cache if available without [`refresh_key`][ref-schema-cube-ref-refresh-key] check to renew in foreground. | `false` | +| `queryCacheOptions.queueOptions` | Query queue options for DB queries | `{}` | +| `preAggregationsOptions` | Query cache options for pre-aggregations | `{}` | +| `preAggregationsOptions.maxPartitions` | The maximum number of partitions each pre-aggregation in a cube can use. | `10000` | +| `preAggregationsOptions.queueOptions` | Query queue options for pre-aggregations | `{}` | +| `preAggregationsOptions.externalRefresh` | When running a separate instance of Cube to refresh pre-aggregations in the background, this option can be set on the API instance to prevent it from trying to check for rollup data being current - it won't try to create or refresh them when this option is `true` | `false` | `queryCacheOptions` are used while querying database tables, while `preAggregationsOptions` settings are used to query pre-aggregated tables. @@ -466,19 +470,19 @@ You can pass this object to set advanced options for the query orchestrator. Setting these options is highly discouraged as these are considered to be -system-level settings. Please use `CUBEJS_DB_QUERY_TIMEOUT` and +system-level settings. Please use `CUBEJS_ROLLUP_ONLY`, `CUBEJS_DB_QUERY_TIMEOUT`, and `CUBEJS_CONCURRENCY` [environment variables][ref-environment-variables] instead. Timeout and interval options' values are in seconds. -| Option | Description | Default Value | -| ----------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------- | -| concurrency | Maximum number of queries to be processed simultaneosly. For drivers with connection pool `CUBEJS_DB_MAX_POOL` should be adjusted accordingly. Typically pool size should be at least twice of total concurrency among all queues. | `2` | -| executionTimeout | Total timeout of single query | `600` | -| orphanedTimeout | Query will be marked for cancellation if not requested during this period. | `120` | -| heartBeatInterval | Worker heartbeat interval. If `4*heartBeatInterval` time passes without reporting, the query gets cancelled. | `30` | +| Option | Description | Default Value | +| ------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------- | +| `concurrency` | Maximum number of queries to be processed simultaneosly. For drivers with connection pool `CUBEJS_DB_MAX_POOL` should be adjusted accordingly. Typically pool size should be at least twice of total concurrency among all queues. | `2` | +| `executionTimeout` | Total timeout of single query | `600` | +| `orphanedTimeout` | Query will be marked for cancellation if not requested during this period. | `120` | +| `heartBeatInterval` | Worker heartbeat interval. If `4*heartBeatInterval` time passes without reporting, the query gets cancelled. | `30` | @@ -554,7 +558,8 @@ the schema name dynamically depending on the security context. Defaults to `dev_pre_aggregations` in [development mode][ref-development-mode] and `prod_pre_aggregations` in production. -Can be also set via the `CUBEJS_PRE_AGGREGATIONS_SCHEMA` environment variable. +This configuration option can also be set using the `CUBEJS_PRE_AGGREGATIONS_SCHEMA` +environment variable. @@ -641,8 +646,8 @@ Best practice is to run `scheduled_refresh_timer` in a separate worker Cube instance. You may also need to configure -[`scheduledRefreshTimeZones`][self-opts-sched-refresh-tz] and -[`scheduledRefreshContexts`][self-opts-sched-refresh-ctxs]. +[`scheduledRefreshTimeZones`](#scheduled_refresh_time_zones) and +[`scheduledRefreshContexts`](#scheduled_refresh_contexts). ### `scheduled_refresh_time_zones` @@ -676,7 +681,7 @@ module.exports = { The default value is a list of a single time zone. `UTC`. -This configuration option can be also set using the +This configuration option can also be set using the `CUBEJS_SCHEDULED_REFRESH_TIMEZONES` environment variable. ### `scheduled_refresh_contexts` @@ -842,7 +847,7 @@ module.exports = { -This configuration option can be also set using the +This configuration option can also be set using the `CUBEJS_ALLOW_UNGROUPED_WITHOUT_PRIMARY_KEY` environment variable. @@ -1157,20 +1162,15 @@ Please [track this issue](https://github.com/cube-js/cube/issues/8136). }; ``` -* `jwkUrl` — the URL from which JSON Web Key Sets (JWKS) can be retrieved. Can also be set -using `CUBEJS_JWK_URL`. -* `key` — a JSON string that represents a cryptographic key. Similar to `API_SECRET`. Can -also be set using `CUBEJS_JWT_KEY`. -* `algorithms` — [any supported algorithm for decoding JWTs][gh-jsonwebtoken-algs]. Can also be -set using `CUBEJS_JWT_ALGS`. -* `issuer` — an issuer value which will be used to enforce the [`iss` claim from inbound -JWTs][link-jwt-ref-iss]. Can also be set using `CUBEJS_JWT_ISSUER`. -* `audience` — an audience value which will be used to enforce the [`aud` claim from inbound -JWTs][link-jwt-ref-aud]. Can also be set using `CUBEJS_JWT_AUDIENCE`. -* `subject` — a subject value which will be used to enforce the [`sub` claim from inbound -JWTs][link-jwt-ref-sub]. Can also be set using `CUBEJS_JWT_SUBJECT`. -* `claimsNamespace` — a namespace within the decoded JWT under which any custom claims can be found. -Can also be set using `CUBEJS_JWT_CLAIMS_NAMESPACE`. +| Option | Description | Environment variable | +| ------ | ----------- | -------------------- | +| `jwkUrl` | URL from which JSON Web Key Sets (JWKS) can be retrieved | Can also be set using `CUBEJS_JWK_URL` | +| `key` | JSON string that represents a cryptographic key. Similar to `CUBEJS_API_SECRET` | Can also be set using `CUBEJS_JWT_KEY` | +| `algorithms` | [Any supported algorithm for decoding JWTs][gh-jsonwebtoken-algs] | Can also be set using `CUBEJS_JWT_ALGS` | +| `issuer` | Issuer value which will be used to enforce the [`iss` claim from inbound JWTs][link-jwt-ref-iss] | Can also be set using `CUBEJS_JWT_ISSUER` | +| `audience` | Audience value which will be used to enforce the [`aud` claim from inbound JWTs][link-jwt-ref-aud] | Can also be set using `CUBEJS_JWT_AUDIENCE` | +| `subject` | Subject value which will be used to enforce the [`sub` claim from inbound JWTs][link-jwt-ref-sub] | Can also be set using `CUBEJS_JWT_SUBJECT` | +| `claimsNamespace` | Namespace within the decoded JWT under which any custom claims can be found | Can also be set using `CUBEJS_JWT_CLAIMS_NAMESPACE` | ### `check_sql_auth` @@ -1236,9 +1236,8 @@ using `check_sql_auth` to authenticate requests to the SQL API with LDAP. ### `can_switch_sql_user` -Used in the [SQL API][ref-sql-api],. Default -implementation depends on `CUBEJS_SQL_SUPER_USER` and return `true` when it's -equal to session's user. +Used in the [SQL API][ref-sql-api]. Default implementation depends on +`CUBEJS_SQL_SUPER_USER` and returns `true` when it's equal to session's user. Called on each change request from Cube SQL API. @@ -1310,6 +1309,8 @@ module.exports = { +See also the `CUBEJS_LOG_LEVEL` environment variable. + ### `telemetry` Cube collects high-level anonymous usage statistics for servers started in @@ -1394,8 +1395,6 @@ If not defined, Cube will lookup for environment variable [self-opts-ctx-to-appid]: #context_to_app_id [self-driver-factory]: #driver_factory [ref-schema-ref-datasource]: /reference/data-model/cube#data_source -[self-opts-sched-refresh-ctxs]: #scheduled_refresh_contexts -[self-opts-sched-refresh-tz]: #scheduled_refresh_time_zones [self-repofactory]: #repository_factory [self-schemafilerepo]: #schema_file_repository [self-schemapath]: #schema_path diff --git a/docs/pages/reference/configuration/environment-variables.mdx b/docs/pages/reference/configuration/environment-variables.mdx index 044efede3b1e6..39f3f1819b7de 100644 --- a/docs/pages/reference/configuration/environment-variables.mdx +++ b/docs/pages/reference/configuration/environment-variables.mdx @@ -21,6 +21,9 @@ The secret key used to sign and verify JWTs. Generated on project scaffold with | --------------- | ---------------------- | --------------------- | | A valid string | N/A | N/A | +See also the [`check_auth` configuration +option](/reference/configuration/config#check_auth). + ## `CUBEJS_APP` An application ID used to uniquely identify the Cube deployment. Can be @@ -96,6 +99,9 @@ The cache and queue driver to use for the Cube deployment. | --------------------- | ---------------------- | --------------------- | | `cubestore`, `memory` | `memory` | `cubestore` | +It can be also set using the [`cache_and_queue_driver` configuration +option](/reference/configuration/config#cache_and_queue_driver). + ## `CUBEJS_CONCURRENCY` The number of concurrent connections each query queue has to the database. @@ -104,6 +110,9 @@ The number of concurrent connections each query queue has to the database. | --------------- | ------------------------------------------- | ------------------------------------------- | | A valid number | [See database-specific page][ref-config-db] | [See database-specific page][ref-config-db] | +It can be also set as `concurrency` in the [`orchestrator_options` configuration +option](/reference/configuration/config#orchestrator_options). + ## `CUBEJS_CUBESTORE_HOST` The hostname of the Cube Store deployment @@ -718,6 +727,9 @@ endpoints. | ------------------------------------------------------------------------------ | ---------------------- | --------------------- | | A comma-delimited string with any combination of [API scopes][ref-rest-scopes] | `meta,data,graphql` | `meta,data,graphql` | +See also the [`context_to_api_scopes` configuration +option](/reference/configuration/config#context_to_api_scopes). + ## `CUBEJS_DEV_MODE` If `true`, enables [development mode](/product/configuration#development-mode). @@ -782,6 +794,9 @@ Enables [JSON Web Key (JWK)][ietf-jwk-ref]-based authentication in Cube. | --------------------------------- | ---------------------- | --------------------- | | A valid URL to a JSON Web Key Set | N/A | N/A | +It can be also set as `jwkUrl` in the [`jwt` configuration +option](/reference/configuration/config#jwt). + ## `CUBEJS_JWT_ALGS` [Any supported algorithm for decoding JWTs][gh-jsonwebtoken-algs]. @@ -790,6 +805,9 @@ Enables [JSON Web Key (JWK)][ietf-jwk-ref]-based authentication in Cube. | ---------------- | ---------------------- | --------------------- | | `HS256`, `RS256` | N/A | N/A | +It can be also set as `algorithms` in the [`jwt` configuration +option](/reference/configuration/config#jwt). + ## `CUBEJS_JWT_AUDIENCE` An audience value which will be used to enforce the [`aud` claim from inbound @@ -799,6 +817,9 @@ JWTs][ietf-jwt-ref-aud]. | ------------------- | ---------------------- | --------------------- | | A valid `aud` claim | N/A | N/A | +It can be also set as `audience` in the [`jwt` configuration +option](/reference/configuration/config#jwt). + ## `CUBEJS_JWT_CLAIMS_NAMESPACE` A namespace within the decoded JWT under which any custom claims can be found. @@ -807,6 +828,9 @@ A namespace within the decoded JWT under which any custom claims can be found. | --------------- | ---------------------- | --------------------- | | A valid string | N/A | N/A | +It can be also set as `claimsNamespace` in the [`jwt` configuration +option](/reference/configuration/config#jwt). + ## `CUBEJS_JWT_ISSUER` An issuer value which will be used to enforce the [`iss` claim from inbound @@ -816,6 +840,9 @@ JWTs][ietf-jwt-ref-iss]. | ------------------- | ---------------------- | --------------------- | | A valid `iss` claim | N/A | N/A | +It can be also set as `issuer` in the [`jwt` configuration +option](/reference/configuration/config#jwt). + ## `CUBEJS_JWT_KEY` The secret key used to sign and verify JWTs. Similar to @@ -825,6 +852,9 @@ The secret key used to sign and verify JWTs. Similar to | --------------- | ---------------------- | --------------------- | | A valid string | N/A | N/A | +It can be also set as `key` in the [`jwt` configuration +option](/reference/configuration/config#jwt). + ## `CUBEJS_JWT_SUBJECT` A subject value which will be used to enforce the [`sub` claim from inbound @@ -834,6 +864,9 @@ JWTs][ietf-jwt-ref-sub]. | ------------------- | ---------------------- | --------------------- | | A valid `sub` claim | N/A | N/A | +It can be also set as `subject` in the [`jwt` configuration +option](/reference/configuration/config#jwt). + ## `CUBEJS_LOG_LEVEL` The logging level for Cube. @@ -842,6 +875,9 @@ The logging level for Cube. | -------------------------------- | ---------------------- | --------------------- | | `error`, `info`, `trace`, `warn` | `warn` | `warn` | +See also `CUBESTORE_LOG_LEVEL`. +See also the [`logger` configuration option](/reference/configuration/config#logger). + ## `CUBEJS_MAX_PARTITIONS_PER_CUBE` The maximum number of partitions each pre-aggregation in a cube can use. @@ -883,6 +919,9 @@ to use for storing pre-aggregations. | --------------- | ---------------------- | ----------------------- | | A valid string | `dev_pre_aggregations` | `prod_pre_aggregations` | +It can be also set using the [`pre_aggregations_schema` configuration +option](/reference/configuration/config#pre_aggregations_schema). + ## `CUBEJS_REFRESH_WORKER` If `true`, this instance of Cube will **only** refresh pre-aggregations. @@ -901,6 +940,9 @@ mode](/product/caching/using-pre-aggregations#rollup-only-mode) for details. | --------------- | ---------------------- | --------------------- | | `true`, `false` | `false` | `false` | +It can be also set using the [`orchestrator_options.rollupOnlyMode` configuration +option](/reference/configuration/config#orchestrator_options). + ## `CUBEJS_SCHEDULED_REFRESH_CONCURRENCY` How many pre-aggregations refresh worker will build in parallel. Please note @@ -920,6 +962,9 @@ for][ref-config-sched-refresh-timer]. | --------------------------------------------------------- | ---------------------- | --------------------- | | [A valid timezone from the tz database][wiki-tz-database] | N/A | N/A | +It can be also set using the [`scheduled_refresh_time_zones` configuration +option](/reference/configuration/config#scheduled_refresh_time_zones). + ## `CUBEJS_SCHEMA_PATH` The path where Cube loads data models from. @@ -928,38 +973,49 @@ The path where Cube loads data models from. | ---------------------------------------- | ---------------------- | --------------------- | | A valid path containing Cube data models | `model` | `model` | - + Until v0.35, the default value was `schema`. - + -{/* TODO: https://cubedevinc.atlassian.net/browse/CC-3095 */} +It can be also set using the [`schema_path` configuration +option](/reference/configuration/config#schema_path). -## `CUBEJS_SQL_PASSWORD` +## `CUBEJS_SQL_USER` -A password required to access the [SQL API][ref-sql-api]. +A username required to access the [SQL API][ref-sql-api]. | Possible Values | Default in Development | Default in Production | | --------------- | ---------------------- | --------------------- | | A valid string | N/A | N/A | -## `CUBEJS_SQL_SUPER_USER` +See also the [`check_sql_auth` configuration +option](/reference/configuration/config#check_sql_auth). + +## `CUBEJS_SQL_PASSWORD` -A name of specific user who will be allowed to change security context. +A password required to access the [SQL API][ref-sql-api]. | Possible Values | Default in Development | Default in Production | | --------------- | ---------------------- | --------------------- | | A valid string | N/A | N/A | -## `CUBEJS_SQL_USER` +See also the [`check_sql_auth` configuration +option](/reference/configuration/config#check_sql_auth). -A username required to access the [SQL API][ref-sql-api]. +## `CUBEJS_SQL_SUPER_USER` + +A name of specific user who will be allowed to change the user during the SQL API +session. | Possible Values | Default in Development | Default in Production | | --------------- | ---------------------- | --------------------- | | A valid string | N/A | N/A | +See also the [`can_switch_sql_user` configuration +option](/reference/configuration/config#can_switch_sql_user). + ## `CUBEJS_ALLOW_UNGROUPED_WITHOUT_PRIMARY_KEY` If `true`, disables the primary key inclusion check for @@ -1220,6 +1276,8 @@ The logging level for Cube Store. | ----------------------------------------- | ---------------------- | --------------------- | | `error`, `warn`, `info`, `debug`, `trace` | `error` | `error` | +See also `CUBEJS_LOG_LEVEL`. + ## `CUBESTORE_META_ADDR` The address/port pair for the Cube Store **router** node in the cluster. From 2514dae4264b8d5dbe3183902f8abfab857a0555 Mon Sep 17 00:00:00 2001 From: Igor Lukanin Date: Mon, 2 Dec 2024 14:46:59 +0100 Subject: [PATCH 08/16] docs: Multi-tenant scheduled_refresh_time_zones --- .../configuration/advanced/multitenancy.mdx | 23 +++++++----- docs/pages/reference/configuration/config.mdx | 37 +++++++++++++++++-- 2 files changed, 46 insertions(+), 14 deletions(-) diff --git a/docs/pages/product/configuration/advanced/multitenancy.mdx b/docs/pages/product/configuration/advanced/multitenancy.mdx index ca4e152a84a3b..937055d1d5b60 100644 --- a/docs/pages/product/configuration/advanced/multitenancy.mdx +++ b/docs/pages/product/configuration/advanced/multitenancy.mdx @@ -9,16 +9,19 @@ Cube supports multitenancy out of the box, both on database and data model levels. Multiple drivers are also supported, meaning that you can have one customer’s data in MongoDB and others in Postgres with one Cube instance. -There are 6 [configuration options][ref-config-opts] you can leverage to make +There are several [configuration options][ref-config-opts] you can leverage for your multitenancy setup. You can use all of them or just a couple, depending on your specific case. The options are: -- `contextToAppId` -- `contextToOrchestratorId` -- `driverFactory` -- `repositoryFactory` -- `preAggregationsSchema` -- `queryRewrite` +- `context_to_app_id` +- `schema_version` +- `repository_factory` +- `driver_factory` +- `context_to_orchestrator_id` +- `pre_aggregations_schema` +- `query_rewrite` +- `scheduled_refresh_contexts` +- `scheduled_refresh_time_zones` All of the above options are functions, which you provide to Cube in the [`cube.js` configuration file][ref-config]. The functions accept one argument - @@ -360,12 +363,12 @@ module.exports = { If you need scheduled refreshes for your pre-aggregations in a multi-tenant deployment, ensure you have configured -[`scheduledRefreshContexts`][ref-config-refresh-ctx] correctly. You may also -need to configure [`scheduledRefreshTimeZones`][ref-config-refresh-tz]. +[`scheduled_refresh_contexts`][ref-config-refresh-ctx] correctly. You may also +need to configure [`scheduled_refresh_time_zones`][ref-config-refresh-tz]. -Leaving [`scheduledRefreshContexts`][ref-config-refresh-ctx] unconfigured will +Leaving [`scheduled_refresh_contexts`][ref-config-refresh-ctx] unconfigured will lead to issues where the security context will be `undefined`. This is because there is no way for Cube to know how to generate a context without the required input. diff --git a/docs/pages/reference/configuration/config.mdx b/docs/pages/reference/configuration/config.mdx index 886207887fa25..b9abd79237c33 100644 --- a/docs/pages/reference/configuration/config.mdx +++ b/docs/pages/reference/configuration/config.mdx @@ -654,32 +654,61 @@ You may also need to configure This option specifies a list of time zones that pre-aggregations will be built for. It has impact on [pre-aggregation matching][ref-matching-preaggs]. -You can specify multiple timezones in the [TZ Database Name][link-wiki-tz] -format, e.g., `America/Los_Angeles`: +Either an array or function returning an array can be passed. Providing a function +allows to set the time zones dynamically depending on the security context. + +Time zones should be specified in the [TZ Database Name][link-wiki-tz] format, +e.g., `America/Los_Angeles`. ```python from cube import config +# An array of time zones config.scheduled_refresh_time_zones = [ 'America/Vancouver', 'America/Toronto' ] + +# Alternatively, a function returning an array of time zones +@config('scheduled_refresh_time_zones') +def scheduled_refresh_time_zones(ctx: dict) -> list[str]: + time_zones = { + 'tenant_1': ['America/New_York'], + 'tenant_2': ['America/Chicago'], + 'tenant_3': ['America/Los_Angeles'] + } + default_time_zones = ['UTC'] + tenant_id = ctx['securityContext']['tenant_id'] + return time_zones.get(tenant_id, default_time_zones) ``` ```javascript module.exports = { + // An array of time zones scheduledRefreshTimeZones: [ 'America/Vancouver', 'America/Toronto' - ] + ], + + // Alternatively, a function returning an array of time zones + scheduledRefreshTimeZones: ({ securityContext }) => { + let time_zones = { + 'tenant_1': ['America/New_York'], + 'tenant_2': ['America/Chicago'], + 'tenant_3': ['America/Los_Angeles'] + } + let default_time_zones = ['UTC'] + let tenant_id = securityContext.tenant_id + return time_zones[tenant_id] || default_time_zones + } }; ``` -The default value is a list of a single time zone. `UTC`. +The default value is a list of a single time zone: `UTC`. This configuration option can also be set using the `CUBEJS_SCHEDULED_REFRESH_TIMEZONES` environment variable. From 57ecdce55c7194675330429c1cb018788719545c Mon Sep 17 00:00:00 2001 From: Igor Lukanin Date: Mon, 9 Dec 2024 10:58:35 +0100 Subject: [PATCH 09/16] Fix --- docs/pages/product/configuration/advanced/multitenancy.mdx | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/pages/product/configuration/advanced/multitenancy.mdx b/docs/pages/product/configuration/advanced/multitenancy.mdx index 937055d1d5b60..eb8c1cda9b375 100644 --- a/docs/pages/product/configuration/advanced/multitenancy.mdx +++ b/docs/pages/product/configuration/advanced/multitenancy.mdx @@ -23,8 +23,8 @@ your specific case. The options are: - `scheduled_refresh_contexts` - `scheduled_refresh_time_zones` -All of the above options are functions, which you provide to Cube in the -[`cube.js` configuration file][ref-config]. The functions accept one argument - +All of the above options are functions, which you provide in the +[configuration file][ref-config]. The functions accept one argument: a context object, which has a [`securityContext`][ref-config-security-ctx] property where you can provide all the necessary data to identify a user e.g., organization, app, etc. By default, the @@ -375,7 +375,7 @@ input. -[ref-config]: /reference/configuration/config +[ref-config]: /product/configuration#configuration-options [ref-config-opts]: /reference/configuration/config [ref-config-db]: /product/configuration/data-sources [ref-config-driverfactory]: /reference/configuration/config#driver_factory From b2a490d8d1240713ff6c866166d0748676241e79 Mon Sep 17 00:00:00 2001 From: Konstantin Burkalev Date: Mon, 9 Dec 2024 15:35:20 +0200 Subject: [PATCH 10/16] =?UTF-8?q?add=20scheduledRefreshTimeZones=20func=20?= =?UTF-8?q?support=20to=C2=A0python?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit cover with tests --- packages/cubejs-backend-native/js/index.ts | 2 ++ .../python/cube/src/__init__.py | 4 +-- packages/cubejs-backend-native/test/config.py | 28 ++++++++++++++++ .../cubejs-backend-native/test/python.test.ts | 32 +++++++++++++++++++ 4 files changed, 64 insertions(+), 2 deletions(-) diff --git a/packages/cubejs-backend-native/js/index.ts b/packages/cubejs-backend-native/js/index.ts index f484c2d6080b0..15fc09da501b3 100644 --- a/packages/cubejs-backend-native/js/index.ts +++ b/packages/cubejs-backend-native/js/index.ts @@ -354,6 +354,8 @@ export interface PyConfiguration { checkAuth?: (req: unknown, authorization: string) => Promise<{ 'security_context'?: unknown }> queryRewrite?: (query: unknown, ctx: unknown) => Promise contextToApiScopes?: () => Promise + scheduledRefreshContexts?: () => Promise + scheduledRefreshTimeZones?: () => Promise contextToRoles?: (ctx: unknown) => Promise } diff --git a/packages/cubejs-backend-native/python/cube/src/__init__.py b/packages/cubejs-backend-native/python/cube/src/__init__.py index 79b1b50de4863..ca87fdf752f93 100644 --- a/packages/cubejs-backend-native/python/cube/src/__init__.py +++ b/packages/cubejs-backend-native/python/cube/src/__init__.py @@ -44,7 +44,7 @@ class Configuration: allow_js_duplicate_props_in_schema: bool jwt: Dict scheduled_refresh_timer: Any - scheduled_refresh_timezones: list[str] + scheduled_refresh_timezones: Union[Callable[[RequestContext], list[str]], list[str]] scheduled_refresh_concurrency: int scheduled_refresh_batch_size: int compiler_cache_size: int @@ -93,7 +93,6 @@ def __init__(self): self.process_subscriptions_interval = None self.jwt = None self.scheduled_refresh_timer = None - self.scheduled_refresh_timezones = None self.scheduled_refresh_concurrency = None self.scheduled_refresh_batch_size = None self.compiler_cache_size = None @@ -118,6 +117,7 @@ def __init__(self): self.query_rewrite = None self.extend_context = None self.scheduled_refresh_contexts = None + self.scheduled_refresh_timezones = None self.context_to_api_scopes = None self.repository_factory = None self.schema_version = None diff --git a/packages/cubejs-backend-native/test/config.py b/packages/cubejs-backend-native/test/config.py index d5f17fed9173f..29d02b4e01812 100644 --- a/packages/cubejs-backend-native/test/config.py +++ b/packages/cubejs-backend-native/test/config.py @@ -33,6 +33,34 @@ async def context_to_api_scopes(): return ["meta", "data", "jobs"] +@config +async def scheduled_refresh_time_zones(ctx): + print("[python] scheduled_refresh_time_zones ctx=", ctx) + return ["Europe/Kyiv", "Antarctica/Troll", "Australia/Sydney"] + + +@config +async def scheduled_refresh_contexts(ctx): + print("[python] scheduled_refresh_contexts ctx=", ctx) + return [ + { + "securityContext": { + "appid": 'test1', "u": { "prop1": "value1" } + } + }, + { + "securityContext": { + "appid": 'test2', "u": { "prop1": "value2" } + } + }, + { + "securityContext": { + "appid": 'test3', "u": { "prop1": "value3" } + } + }, + ] + + @config def schema_version(ctx): print("[python] schema_version", ctx) diff --git a/packages/cubejs-backend-native/test/python.test.ts b/packages/cubejs-backend-native/test/python.test.ts index d2fa11de9e363..93462fd1d7f75 100644 --- a/packages/cubejs-backend-native/test/python.test.ts +++ b/packages/cubejs-backend-native/test/python.test.ts @@ -83,6 +83,38 @@ suite('Python Config', () => { expect(await config.contextToApiScopes()).toEqual(['meta', 'data', 'jobs']); }); + test('scheduled_refresh_time_zones', async () => { + if (!config.scheduledRefreshTimeZones) { + throw new Error('scheduledRefreshTimeZones was not defined in config.py'); + } + + expect(await config.scheduledRefreshTimeZones()).toEqual(['Europe/Kyiv', 'Antarctica/Troll', 'Australia/Sydney']); + }); + + test('scheduled_refresh_contexts', async () => { + if (!config.scheduledRefreshContexts) { + throw new Error('scheduledRefreshContexts was not defined in config.py'); + } + + expect(await config.scheduledRefreshContexts()).toEqual([ + { + securityContext: { + appid: 'test1', u: { prop1: 'value1' } + } + }, + { + securityContext: { + appid: 'test2', u: { prop1: 'value2' } + } + }, + { + securityContext: { + appid: 'test3', u: { prop1: 'value3' } + } + }, + ]); + }); + test('repository factory', async () => { if (!config.repositoryFactory) { throw new Error('repositoryFactory was not defined in config.py'); From 720df5c026790c0d499ca2fb462de032d817b5a7 Mon Sep 17 00:00:00 2001 From: Konstantin Burkalev Date: Mon, 9 Dec 2024 16:51:13 +0200 Subject: [PATCH 11/16] add and debug tests --- packages/cubejs-backend-native/js/index.ts | 4 +-- packages/cubejs-backend-native/package.json | 3 +- .../python/cube/src/__init__.py | 4 +-- .../src/python/cube_config.rs | 2 +- .../cubejs-backend-native/test/old-config.py | 29 +++++++++++++++++++ .../cubejs-backend-native/test/python.test.ts | 9 ++++-- 6 files changed, 43 insertions(+), 8 deletions(-) diff --git a/packages/cubejs-backend-native/js/index.ts b/packages/cubejs-backend-native/js/index.ts index 15fc09da501b3..8c77d1c236099 100644 --- a/packages/cubejs-backend-native/js/index.ts +++ b/packages/cubejs-backend-native/js/index.ts @@ -354,8 +354,8 @@ export interface PyConfiguration { checkAuth?: (req: unknown, authorization: string) => Promise<{ 'security_context'?: unknown }> queryRewrite?: (query: unknown, ctx: unknown) => Promise contextToApiScopes?: () => Promise - scheduledRefreshContexts?: () => Promise - scheduledRefreshTimeZones?: () => Promise + scheduledRefreshContexts?: (ctx: unknown) => Promise + scheduledRefreshTimeZones?: (ctx: unknown) => Promise contextToRoles?: (ctx: unknown) => Promise } diff --git a/packages/cubejs-backend-native/package.json b/packages/cubejs-backend-native/package.json index 153c99c3fefb8..3fefede44c3e4 100644 --- a/packages/cubejs-backend-native/package.json +++ b/packages/cubejs-backend-native/package.json @@ -21,7 +21,8 @@ "test:server": "CUBEJS_NATIVE_INTERNAL_DEBUG=true CUBESQL_LOG_LEVEL=trace CUBESQL_PG_PORT=5555 node dist/test/server.js", "test:server:stream": "CUBESQL_STREAM_MODE=true CUBESQL_LOG_LEVEL=error CUBESQL_PG_PORT=5555 node dist/test/server.js", "test:python": "CUBEJS_NATIVE_INTERNAL_DEBUG=true CUBESQL_LOG_LEVEL=trace CUBESQL_PG_PORT=5555 node dist/test/python.js", - "test:unit": "jest --forceExit", + "unit": "jest --forceExit", + "test:unit": "yarn run unit", "test:cargo": "cargo test", "lint": "eslint test/ js/ --ext .ts", "lint:fix": "eslint --fix test/ js/ --ext .ts" diff --git a/packages/cubejs-backend-native/python/cube/src/__init__.py b/packages/cubejs-backend-native/python/cube/src/__init__.py index ca87fdf752f93..e8b52ab0465b8 100644 --- a/packages/cubejs-backend-native/python/cube/src/__init__.py +++ b/packages/cubejs-backend-native/python/cube/src/__init__.py @@ -44,7 +44,7 @@ class Configuration: allow_js_duplicate_props_in_schema: bool jwt: Dict scheduled_refresh_timer: Any - scheduled_refresh_timezones: Union[Callable[[RequestContext], list[str]], list[str]] + scheduled_refresh_time_zones: Union[Callable[[RequestContext], list[str]], list[str]] scheduled_refresh_concurrency: int scheduled_refresh_batch_size: int compiler_cache_size: int @@ -117,7 +117,7 @@ def __init__(self): self.query_rewrite = None self.extend_context = None self.scheduled_refresh_contexts = None - self.scheduled_refresh_timezones = None + self.scheduled_refresh_time_zones = None self.context_to_api_scopes = None self.repository_factory = None self.schema_version = None diff --git a/packages/cubejs-backend-native/src/python/cube_config.rs b/packages/cubejs-backend-native/src/python/cube_config.rs index 35452af425c13..b0bec3d7224ed 100644 --- a/packages/cubejs-backend-native/src/python/cube_config.rs +++ b/packages/cubejs-backend-native/src/python/cube_config.rs @@ -33,7 +33,7 @@ impl CubeConfigPy { "scheduled_refresh_batch_size", "scheduled_refresh_concurrency", "scheduled_refresh_timer", - "scheduled_refresh_timezones", + "scheduled_refresh_time_zones", "schema_path", "sql_cache", "sql_password", diff --git a/packages/cubejs-backend-native/test/old-config.py b/packages/cubejs-backend-native/test/old-config.py index a84df4bdfffb2..95c7d547f7141 100644 --- a/packages/cubejs-backend-native/test/old-config.py +++ b/packages/cubejs-backend-native/test/old-config.py @@ -31,6 +31,35 @@ async def context_to_api_scopes(): settings.context_to_api_scopes = context_to_api_scopes +async def scheduled_refresh_time_zones(ctx): + print("[python] scheduled_refresh_time_zones ctx=", ctx) + return ["Europe/Kyiv", "Antarctica/Troll", "Australia/Sydney"] + +settings.scheduled_refresh_time_zones = scheduled_refresh_time_zones + + +async def scheduled_refresh_contexts(ctx): + print("[python] scheduled_refresh_contexts ctx=", ctx) + return [ + { + "securityContext": { + "appid": 'test1', "u": {"prop1": "value1"} + } + }, + { + "securityContext": { + "appid": 'test2', "u": {"prop1": "value2"} + } + }, + { + "securityContext": { + "appid": 'test3', "u": {"prop1": "value3"} + } + }, + ] + +settings.scheduled_refresh_contexts = scheduled_refresh_contexts + def schema_version(ctx): print('[python] schema_version', ctx) diff --git a/packages/cubejs-backend-native/test/python.test.ts b/packages/cubejs-backend-native/test/python.test.ts index 93462fd1d7f75..e6f15a996452b 100644 --- a/packages/cubejs-backend-native/test/python.test.ts +++ b/packages/cubejs-backend-native/test/python.test.ts @@ -47,6 +47,8 @@ suite('Python Config', () => { repositoryFactory: expect.any(Function), schemaVersion: expect.any(Function), contextToRoles: expect.any(Function), + scheduledRefreshContexts: expect.any(Function), + scheduledRefreshTimeZones: expect.any(Function), }); if (!config.checkAuth) { @@ -88,7 +90,7 @@ suite('Python Config', () => { throw new Error('scheduledRefreshTimeZones was not defined in config.py'); } - expect(await config.scheduledRefreshTimeZones()).toEqual(['Europe/Kyiv', 'Antarctica/Troll', 'Australia/Sydney']); + expect(await config.scheduledRefreshTimeZones({})).toEqual(['Europe/Kyiv', 'Antarctica/Troll', 'Australia/Sydney']); }); test('scheduled_refresh_contexts', async () => { @@ -96,7 +98,7 @@ suite('Python Config', () => { throw new Error('scheduledRefreshContexts was not defined in config.py'); } - expect(await config.scheduledRefreshContexts()).toEqual([ + expect(await config.scheduledRefreshContexts({})).toEqual([ { securityContext: { appid: 'test1', u: { prop1: 'value1' } @@ -192,6 +194,9 @@ darwinSuite('Old Python Config', () => { queryRewrite: expect.any(Function), repositoryFactory: expect.any(Function), schemaVersion: expect.any(Function), + contextToRoles: expect.any(Function), + scheduledRefreshContexts: expect.any(Function), + scheduledRefreshTimeZones: expect.any(Function), }); if (!config.checkAuth) { From bcd76d7cab33552c394c008510f57dd2302fe590 Mon Sep 17 00:00:00 2001 From: Konstantin Burkalev Date: Mon, 9 Dec 2024 17:45:05 +0200 Subject: [PATCH 12/16] =?UTF-8?q?add=20test=20for=C2=A0string=20config=20o?= =?UTF-8?q?ption=20ScheduledRefreshTimeZones?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- packages/cubejs-server-core/test/unit/index.test.ts | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/packages/cubejs-server-core/test/unit/index.test.ts b/packages/cubejs-server-core/test/unit/index.test.ts index a1f1741d38303..5e7726232aa86 100644 --- a/packages/cubejs-server-core/test/unit/index.test.ts +++ b/packages/cubejs-server-core/test/unit/index.test.ts @@ -23,6 +23,10 @@ class CubejsServerCoreOpen extends CubejsServerCore { public isReadyForQueryProcessing = super.isReadyForQueryProcessing; public createOrchestratorApi = super.createOrchestratorApi; + + public pubScheduledRefreshTimeZones(ctx: any) { + return this.scheduledRefreshTimeZones(ctx); + }; } const repositoryWithoutPreAggregations: SchemaFileRepository = { @@ -300,6 +304,7 @@ describe('index.test', () => { const cubejsServerCore = new CubejsServerCoreOpen(options); expect(cubejsServerCore).toBeInstanceOf(CubejsServerCore); + expect(cubejsServerCore.pubScheduledRefreshTimeZones({} as any)).toEqual(['Europe/Moscow']); const createOrchestratorApiSpy = jest.spyOn(cubejsServerCore, 'createOrchestratorApi'); From b5aa80c9e8ffd7a5bbba778bbb7534116e12dc2e Mon Sep 17 00:00:00 2001 From: Konstantin Burkalev Date: Mon, 9 Dec 2024 17:52:15 +0200 Subject: [PATCH 13/16] make linter happy --- packages/cubejs-server-core/src/core/server.ts | 4 ++-- packages/cubejs-server-core/test/unit/index.test.ts | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/cubejs-server-core/src/core/server.ts b/packages/cubejs-server-core/src/core/server.ts index 6f226eac19331..791c40dd3eca6 100644 --- a/packages/cubejs-server-core/src/core/server.ts +++ b/packages/cubejs-server-core/src/core/server.ts @@ -126,7 +126,7 @@ export class CubejsServerCore { protected readonly orchestratorStorage: OrchestratorStorage = new OrchestratorStorage(); - protected repositoryFactory: ((context: RequestContext) => SchemaFileRepository) | (() => FileRepository); + protected repositoryFactory: ((_context: RequestContext) => SchemaFileRepository) | (() => FileRepository); protected contextToDbType: DbTypeAsyncFn; @@ -162,7 +162,7 @@ export class CubejsServerCore { protected apiGatewayInstance: ApiGateway | null = null; - public readonly event: (name: string, props?: object) => Promise; + public readonly event: (_name: string, _props?: object) => Promise; public projectFingerprint: string | null = null; diff --git a/packages/cubejs-server-core/test/unit/index.test.ts b/packages/cubejs-server-core/test/unit/index.test.ts index 5e7726232aa86..5ff8398240763 100644 --- a/packages/cubejs-server-core/test/unit/index.test.ts +++ b/packages/cubejs-server-core/test/unit/index.test.ts @@ -26,7 +26,7 @@ class CubejsServerCoreOpen extends CubejsServerCore { public pubScheduledRefreshTimeZones(ctx: any) { return this.scheduledRefreshTimeZones(ctx); - }; + } } const repositoryWithoutPreAggregations: SchemaFileRepository = { From 8d8dc33adb95aa43f69d1b5677f16d196726fadb Mon Sep 17 00:00:00 2001 From: Konstantin Burkalev Date: Wed, 11 Dec 2024 22:47:52 +0200 Subject: [PATCH 14/16] =?UTF-8?q?add=20building=20backend-native=20for?= =?UTF-8?q?=C2=A0running=20unit=20tests?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/push.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml index 7431cda4d106d..4b07fbf76189a 100644 --- a/.github/workflows/push.yml +++ b/.github/workflows/push.yml @@ -92,6 +92,9 @@ jobs: run: yarn tsc - name: Build client run: yarn build + - name: Build cubejs-backend-native + run: yarn run native:build-debug-python + working-directory: ./packages/cubejs-backend-native - name: Lerna test run: yarn lerna run --concurrency 1 --stream --no-prefix unit # - uses: codecov/codecov-action@v1 From 1217b5469de1065b71392686455e1d95debf4614 Mon Sep 17 00:00:00 2001 From: Konstantin Burkalev Date: Wed, 11 Dec 2024 23:20:11 +0200 Subject: [PATCH 15/16] =?UTF-8?q?fix=20jest=20multiple=20config=20in=C2=A0?= =?UTF-8?q?client-dx?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- packages/cubejs-client-dx/jest.config.js | 6 ------ packages/cubejs-client-dx/package.json | 7 ++++++- 2 files changed, 6 insertions(+), 7 deletions(-) delete mode 100644 packages/cubejs-client-dx/jest.config.js diff --git a/packages/cubejs-client-dx/jest.config.js b/packages/cubejs-client-dx/jest.config.js deleted file mode 100644 index 0042962b9f935..0000000000000 --- a/packages/cubejs-client-dx/jest.config.js +++ /dev/null @@ -1,6 +0,0 @@ -module.exports = { - verbose: true, - transform: { - '^.+\\.js$': 'babel-jest', - }, -}; diff --git a/packages/cubejs-client-dx/package.json b/packages/cubejs-client-dx/package.json index 258f5722fd203..d6e3040f80ecd 100644 --- a/packages/cubejs-client-dx/package.json +++ b/packages/cubejs-client-dx/package.json @@ -37,7 +37,12 @@ "jest": "^27" }, "jest": { - "testEnvironment": "node" + "testEnvironment": "node", + "verbose": true, + "transform": { + "^.+\\.js$": "babel-jest" + } + }, "publishConfig": { "access": "public" From d2ca1d10f7bc28b5d2266327317dcc5fcce84fa5 Mon Sep 17 00:00:00 2001 From: Konstantin Burkalev Date: Wed, 11 Dec 2024 23:24:56 +0200 Subject: [PATCH 16/16] =?UTF-8?q?add=20python=20for=C2=A0unit=20tests=20in?= =?UTF-8?q?=C2=A0backend-native?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/push.yml | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml index 4b07fbf76189a..c5404abf4eb37 100644 --- a/.github/workflows/push.yml +++ b/.github/workflows/push.yml @@ -42,6 +42,7 @@ jobs: matrix: # Current docker version + next LTS node-version: [20.x, 22.x] + python-version: [3.11] fail-fast: false steps: @@ -65,6 +66,10 @@ jobs: uses: actions/setup-node@v4 with: node-version: ${{ matrix.node-version }} + - name: Install Python + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} - name: Get yarn cache directory path id: yarn-cache-dir-path run: echo "dir=$(yarn cache dir)" >> "$GITHUB_OUTPUT" @@ -92,9 +97,12 @@ jobs: run: yarn tsc - name: Build client run: yarn build - - name: Build cubejs-backend-native - run: yarn run native:build-debug-python + - name: Build cubejs-backend-native (with Python) + run: yarn run native:build-release-python working-directory: ./packages/cubejs-backend-native + env: + PYO3_PYTHON: python${{ matrix.python-version }} + - name: Lerna test run: yarn lerna run --concurrency 1 --stream --no-prefix unit # - uses: codecov/codecov-action@v1